vhaudiquet/pkh
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

distro_info: model distro mirrors, components and build profiles in the yaml data

Browse Source
This commit is contained in:
vhaudiquet
2026-09-18 13:56:36 +02:00
parent dd1c70c91a
commit b8b2be5acf
2 changed files with 373 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/distro_info.yml
+48 -2
View File
@@ -6,9 +6,42 @@
## or Debian series, but rather pointers to where that data lives: each dist
## entry below carries its series sources (the local distro-info CSV, with
## the network URL as fallback).
##
## Per-dist keys beyond the series pointers:
## mirrors: the archive mirrors, each a URL serving a set of
## architectures: `primary` (the main archive, whose url
## doubles as the dist's base URL) and, where they exist,
## the others (`ports`). `security_url` is the sibling
## host serving the -security pocket for the same arches
## (ports mirrors serve their own security); `archs` is
## an explicit list, or the `all` sentinel when one
## mirror serves every architecture (Debian's case — an
## exhaustive list would rot each time an arch is added).
## Host matching treats a URI as official when its host
## equals a mirror host or is a subdomain of it, so the
## country mirrors (fr.archive.ubuntu.com) count too.
## components: the archive components (main, universe, contrib, ...)
## a cross-build environment enables on official sources.
## Live archive operations keep resolving components from
## Release files; this is the offline default.
## cross_pockets: the pockets a cross-build environment enables for a
## series (`<series>-updates`, ...). Deliberately not the
## `pockets` key: that one is the *search order* of pull,
## where backports must not fold in.
## build_profiles: the vendor's default DEB_BUILD_PROFILES (Ubuntu
## activates derivative.ubuntu noudeb, Debian none),
## mirroring what Dpkg::BuildProfiles resolves when the
## variable is unset.
dist:
debian:
base_url: https://deb.debian.org/debian
mirrors:
primary:
url: https://deb.debian.org/debian
# One mirror serves every architecture.
archs: all
components: [main, contrib, non-free, non-free-firmware]
cross_pockets: [updates, backports, security]
build_profiles: []
archive_keyring: https://ftp-master.debian.org/keys/archive-key-{series_num}.asc
pockets:
- updates
@@ -82,7 +115,20 @@ dist:
local: /usr/share/distro-info/debian.csv
network: https://salsa.debian.org/debian/distro-info-data/-/raw/main/debian.csv
ubuntu:
base_url: https://archive.ubuntu.com/ubuntu
mirrors:
primary:
url: https://archive.ubuntu.com/ubuntu
# Sibling host serving the -security pocket for the same arches.
security_url: http://security.ubuntu.com/ubuntu
archs: [amd64, i386]
ports:
# Everything else lives on the ports archive, which also serves
# its own -security pocket (no security_url needed).
url: http://ports.ubuntu.com/ubuntu-ports
archs: [armhf, arm64, ppc64el, riscv64, s390x]
components: [main, restricted, universe, multiverse]
cross_pockets: [updates, backports, security]
build_profiles: [derivative.ubuntu, noudeb]
archive_keyring: https://archive.ubuntu.com/ubuntu/project/ubuntu-archive-keyring.gpg
pockets:
- updates
src/distro_info.rs
+325 -24
View File
@@ -2,6 +2,7 @@ use crate::data::embed_data;
use chrono::NaiveDate;
use lazy_static::lazy_static;
use serde::Deserialize;
use std::collections::HashMap;
use std::error::Error;
use std::path::Path;
use std::time::Duration;
@@ -29,19 +30,69 @@ struct SeriesInfo {
network: String,
}
/// Architectures an archive mirror serves: an explicit list, or the `all`
/// sentinel meaning one mirror serves every architecture (Debian's mirror
/// setup — an exhaustive list would rot each time an arch is added)
#[derive(Debug, Deserialize)]
#[serde(untagged)]
enum MirrorArchs {
/// The `all` sentinel
All(String),
/// An explicit list of dpkg architecture names
List(Vec<String>),
}
impl MirrorArchs {
/// Whether the mirror serves `arch`. A scalar other than the `all`
/// sentinel serves nothing (a test locks that reading of the data).
fn serves(&self, arch: &str) -> bool {
match self {
MirrorArchs::All(sentinel) => sentinel == "all",
MirrorArchs::List(archs) => archs.iter().any(|a| a == arch),
}
}
}
/// One archive mirror of a distribution: a URL serving a set of
/// architectures, plus the sibling host serving its `-security` pocket
/// for the same architectures (ports mirrors serve their own security)
#[derive(Debug, Deserialize)]
pub struct Mirror {
/// Base URL of the mirror (the primary mirror's URL doubles as the
/// dist's base URL, see [`get_base_url`])
pub url: String,
/// Sibling host serving the `-security` pocket for these
/// architectures; `None` when the mirror serves its own security
#[serde(default)]
security_url: Option<String>,
/// Architectures the mirror serves (see [`MirrorArchs`])
archs: MirrorArchs,
}
impl Mirror {
/// Whether the mirror serves `arch`
pub fn serves(&self, arch: &str) -> bool {
self.archs.serves(arch)
}
}
#[derive(Debug, Deserialize)]
struct DistData {
base_url: String,
mirrors: HashMap<String, Mirror>,
archive_keyring: String,
pockets: Vec<String>,
#[serde(default)]
sections: Vec<String>,
components: Vec<String>,
cross_pockets: Vec<String>,
#[serde(default)]
build_profiles: Vec<String>,
series: SeriesInfo,
}
#[derive(Debug, Deserialize)]
struct Data {
dist: std::collections::HashMap<String, DistData>,
dist: HashMap<String, DistData>,
}
embed_data! {
@@ -220,6 +271,24 @@ pub fn supported_dists() -> Vec<String> {
dists
}
/// Name of a dist's primary mirror entry: its URL doubles as the dist's
/// base URL ([`get_base_url`]) and is the first candidate of
/// [`mirror_for_arch`]
const PRIMARY_MIRROR: &str = "primary";
/// The data of a known distribution: the shared "unknown distribution"
/// error of the per-dist accessors
fn dist_data(dist: &str) -> Result<&'static DistData, Box<dyn Error>> {
DATA.dist.get(dist).ok_or_else(|| {
format!(
"Unknown distribution '{}'. Supported distributions are: {}.",
dist,
supported_dists().join(", ")
)
.into()
})
}
/// Special changelog distribution marking an entry that has not been
/// released to any archive series yet
pub const UNRELEASED: &str = "UNRELEASED";
@@ -352,14 +421,7 @@ pub async fn get_dist_from_series(series: &str) -> Result<String, Box<dyn Error>
///
/// Example: get_dist_pockets(ubuntu) => ["", "updates", "security", "proposed"]
pub fn get_dist_pockets(dist: &str) -> Result<Vec<String>, Box<dyn Error>> {
let dist_data = DATA.dist.get(dist).ok_or_else(|| {
format!(
"Unknown distribution '{}'. Supported distributions are: {}.",
dist,
supported_dists().join(", ")
)
})?;
let mut pockets = dist_data.pockets.clone();
let mut pockets = dist_data(dist)?.pockets.clone();
// Explicitely add 'main' pocket, which is just the empty string, first
pockets.insert(0, "".to_string());
@@ -367,18 +429,38 @@ pub fn get_dist_pockets(dist: &str) -> Result<Vec<String>, Box<dyn Error>> {
Ok(pockets)
}
/// Get the archive components of a distribution (ubuntu's main,
/// restricted, universe, multiverse; Debian's main, contrib, non-free,
/// non-free-firmware): the default set a build environment enables on its
/// official sources. Live archive operations keep resolving components
/// from Release files ([`get_components`]); this is the offline default.
pub fn get_dist_components(dist: &str) -> Result<Vec<String>, Box<dyn Error>> {
Ok(dist_data(dist)?.components.clone())
}
/// Get the pockets a cross-build environment enables for a series (the
/// `<series>-<pocket>` suite list is built from these): updates,
/// backports and security. Deliberately separate from
/// [`get_dist_pockets`], which is the *search order* of pull — folding
/// backports into it would change pull behavior.
pub fn get_cross_pockets(dist: &str) -> Result<Vec<String>, Box<dyn Error>> {
Ok(dist_data(dist)?.cross_pockets.clone())
}
/// Get the default build profiles of a distribution's vendor (Ubuntu
/// activates `derivative.ubuntu noudeb`, Debian none), mirroring what
/// `Dpkg::BuildProfiles` resolves when `DEB_BUILD_PROFILES` is unset.
/// Vendors are matched case-insensitively by the caller (dpkg's `Vendor:`
/// field keeps its original casing).
pub fn get_build_profiles(dist: &str) -> Result<Vec<String>, Box<dyn Error>> {
Ok(dist_data(dist)?.build_profiles.clone())
}
/// Get the valid `Section` values of a distribution's packages, as accepted
/// by its archives (a `section/subsection` in debian/control validates on
/// the part before the '/')
pub fn get_sections(dist: &str) -> Result<Vec<String>, Box<dyn Error>> {
let dist_data = DATA.dist.get(dist).ok_or_else(|| {
format!(
"Unknown distribution '{}'. Supported distributions are: {}.",
dist,
supported_dists().join(", ")
)
})?;
Ok(dist_data.sections.clone())
Ok(dist_data(dist)?.sections.clone())
}
/// Get the sources URL for a distribution, series, pocket, and component
@@ -391,23 +473,105 @@ pub fn get_sources_url(base_url: &str, series: &str, pocket: &str, component: &s
format!("{base_url}/dists/{series}{pocket_full}/{component}/source/Sources.gz")
}
/// Get the archive base URL for a distribution
/// Get the archive base URL for a distribution: the URL of its primary
/// mirror (the former `base_url` key folded into `mirrors.primary.url`
/// when the mirrors were modeled — the signature is kept so the pull
/// paths do not churn)
///
/// Example: ubuntu => https://archive.ubuntu.com/ubuntu
pub fn get_base_url(dist: &str) -> Result<String, Box<dyn Error>> {
DATA.dist
.get(dist)
.map(|d| d.base_url.clone())
let mirror = dist_data(dist)?
.mirrors
.get(PRIMARY_MIRROR)
.ok_or_else(|| {
format!(
"Unknown distribution '{}'. Supported distributions are: {}.",
dist,
"Distribution '{dist}' has no '{PRIMARY_MIRROR}' mirror in the built-in \
configuration. This is a bug; supported distributions are: {}.",
supported_dists().join(", ")
)
})?;
Ok(mirror.url.clone())
}
/// The mirror of `dist` serving `arch`: the primary mirror first, then
/// the other mirrors by name, so the answer is deterministic. Debian's
/// `all` sentinel makes its primary mirror serve every architecture.
/// Errors when the dist is unknown or no mirror serves the architecture
/// (an architecture the built-in data does not know about).
///
/// Example: mirror_for_arch(ubuntu, riscv64) => the ports mirror
pub fn mirror_for_arch(dist: &str, arch: &str) -> Result<&'static Mirror, Box<dyn Error>> {
let data = dist_data(dist)?;
if let Some(primary) = data.mirrors.get(PRIMARY_MIRROR)
&& primary.serves(arch)
{
return Ok(primary);
}
let mut others: Vec<&String> = data
.mirrors
.keys()
.filter(|name| name.as_str() != PRIMARY_MIRROR)
.collect();
others.sort();
others
.into_iter()
.filter_map(|name| data.mirrors.get(name))
.find(|mirror| mirror.serves(arch))
.ok_or_else(|| {
format!(
"No mirror of '{dist}' serves the '{arch}' architecture. Supported \
distributions are: {}.",
supported_dists().join(", ")
)
.into()
})
}
/// Host part of an apt-source URL: everything after the `://` scheme up
/// to the first `/` (a `:port` suffix stripped). URLs without a scheme
/// yield their leading segment.
fn url_host(url: &str) -> &str {
let after_scheme = url.split_once("://").map_or(url, |(_, rest)| rest);
let authority = after_scheme.split('/').next().unwrap_or(after_scheme);
authority
.split_once(':')
.map_or(authority, |(host, _)| host)
}
/// Whether the host of `uri` is the host of `mirror_url` or a subdomain
/// of it: the old substring checks (`uri.contains("archive.ubuntu.com")`)
/// intentionally matched the country mirrors fronting each archive host
/// (`fr.archive.ubuntu.com`), and an exact host comparison would have
/// dropped them. The leading dot of the suffix keeps look-alike hosts
/// (`notarchive.ubuntu.com`) out.
fn uri_matches_url_host(uri: &str, mirror_url: &str) -> bool {
let host = url_host(mirror_url);
let uri_host = url_host(uri);
uri_host == host || uri_host.ends_with(&format!(".{host}"))
}
/// Whether `uri` points at `mirror` or its security sibling: the URI's
/// host is the mirror's (or the sibling's) host or a subdomain of it (see
/// [`uri_matches_url_host`])
pub fn is_mirror_source(mirror: &Mirror, uri: &str) -> bool {
uri_matches_url_host(uri, &mirror.url)
|| mirror
.security_url
.as_deref()
.is_some_and(|security| uri_matches_url_host(uri, security))
}
/// Whether `uri` points at an official source of `dist` — one of its
/// archive mirrors or their security siblings — as opposed to a PPA or
/// another third-party repository. Unknown distributions match nothing.
pub fn is_official_source(dist: &str, uri: &str) -> bool {
DATA.dist.get(dist).is_some_and(|data| {
data.mirrors
.values()
.any(|mirror| is_mirror_source(mirror, uri))
})
}
/// Obtain the URLs for the archive keyrings of a distribution series
///
/// For 'sid' and 'experimental', returns keyrings from the 3 latest releases
@@ -565,6 +729,143 @@ mod tests {
assert!(get_sections("not-a-distro").is_err());
}
/// The primary mirror's URL is the former `base_url`, byte for byte:
/// the pull paths build their archive URLs from it.
#[test]
fn test_primary_mirror_url_is_the_base_url() {
assert_eq!(
get_base_url("ubuntu").unwrap(),
"https://archive.ubuntu.com/ubuntu"
);
assert_eq!(
get_base_url("debian").unwrap(),
"https://deb.debian.org/debian"
);
assert!(get_base_url("not-a-distro").is_err());
}
/// Mirror-per-architecture resolution: the local architectures come
/// from Ubuntu's primary mirror, the others from ports; Debian's `all`
/// sentinel makes its one mirror serve everything, including
/// architectures the data never lists.
#[test]
fn test_mirror_for_arch() {
assert_eq!(
mirror_for_arch("ubuntu", "amd64").unwrap().url,
"https://archive.ubuntu.com/ubuntu"
);
assert_eq!(
mirror_for_arch("ubuntu", "riscv64").unwrap().url,
"http://ports.ubuntu.com/ubuntu-ports"
);
for arch in ["amd64", "riscv64", "brand-new"] {
assert_eq!(
mirror_for_arch("debian", arch).unwrap().url,
"https://deb.debian.org/debian",
"the `all` sentinel serves every architecture, including {arch}"
);
}
// An architecture no Ubuntu mirror serves, and an unknown dist.
assert!(mirror_for_arch("ubuntu", "mips64el").is_err());
assert!(mirror_for_arch("not-a-distro", "amd64").is_err());
}
/// The `archs` forms and their reading: the `all` sentinel serves
/// everything, an explicit list serves exactly its members, and any
/// other scalar serves nothing (a data bug a validation would have to
/// catch, hence the documented reading).
#[test]
fn test_mirror_archs_forms() {
let all: MirrorArchs = serde_yaml::from_str("all").unwrap();
assert!(all.serves("anything"));
let list: MirrorArchs = serde_yaml::from_str("[amd64, i386]").unwrap();
assert!(list.serves("amd64"));
assert!(!list.serves("arm64"));
let typo: MirrorArchs = serde_yaml::from_str("every").unwrap();
assert!(!typo.serves("amd64"));
}
/// Official-source matching is host-based but keeps matching the
/// country mirrors the old substring checks matched (`fr.archive.
/// ubuntu.com`): equality or a `.{host}` suffix, never a bare
/// substring — `notarchive.ubuntu.com` must not match.
#[test]
fn test_is_official_source_matches_country_mirrors_only() {
for uri in [
"https://archive.ubuntu.com/ubuntu",
"http://security.ubuntu.com/ubuntu",
"http://ports.ubuntu.com/ubuntu-ports",
// Country mirrors front the same archives.
"http://fr.archive.ubuntu.com/ubuntu",
"https://de.security.ubuntu.com/ubuntu",
] {
assert!(is_official_source("ubuntu", uri), "{uri}");
}
for uri in [
"https://deb.debian.org/debian",
"https://ppa.launchpadcontent.net/user/ppa/ubuntu",
"http://notarchive.ubuntu.com/ubuntu",
"http://archive.ubuntu.com.evil.example/ubuntu",
] {
assert!(!is_official_source("ubuntu", uri), "{uri}");
}
// Debian: its own mirror matches, Ubuntu's mirrors do not, and an
// unknown dist matches nothing.
assert!(is_official_source(
"debian",
"https://deb.debian.org/debian"
));
assert!(!is_official_source(
"debian",
"http://security.ubuntu.com/ubuntu"
));
assert!(!is_official_source(
"not-a-distro",
"https://deb.debian.org/debian"
));
}
/// The dist-level defaults the build paths read: components,
/// cross-build pockets (deliberately not the pull search order) and
/// vendor build profiles.
#[test]
fn test_dist_components_cross_pockets_and_build_profiles() {
assert_eq!(
get_dist_components("ubuntu").unwrap(),
vec!["main", "restricted", "universe", "multiverse"]
);
assert!(
get_dist_components("debian")
.unwrap()
.contains(&"non-free-firmware".to_string())
);
for dist in ["debian", "ubuntu"] {
assert_eq!(
get_cross_pockets(dist).unwrap(),
vec!["updates", "backports", "security"]
);
}
// Not the pull search order: no 'proposed', no empty main pocket.
let cross = get_cross_pockets("ubuntu").unwrap();
assert!(!cross.contains(&"proposed".to_string()));
assert!(!cross.contains(&"".to_string()));
assert_eq!(
get_build_profiles("ubuntu").unwrap(),
vec!["derivative.ubuntu", "noudeb"]
);
assert!(get_build_profiles("debian").unwrap().is_empty());
for getter in [
get_dist_components as fn(&str) -> Result<Vec<String>, Box<dyn Error>>,
get_cross_pockets as fn(&str) -> Result<Vec<String>, Box<dyn Error>>,
get_build_profiles as fn(&str) -> Result<Vec<String>, Box<dyn Error>>,
] {
assert!(getter("not-a-distro").is_err());
}
}
#[test]
fn test_parse_series_csv_malformed_rows() {
// A short row (missing 'codename') is skipped, a row with an invalid