From b8b2be5acf2cd60dd90ea07036d9d44ff760e406 Mon Sep 17 00:00:00 2001 From: Valentin Haudiquet Date: Fri, 18 Sep 2026 13:56:36 +0200 Subject: [PATCH] distro_info: model distro mirrors, components and build profiles in the yaml data --- data/distro_info.yml | 50 ++++++- src/distro_info.rs | 349 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 373 insertions(+), 26 deletions(-) diff --git a/data/distro_info.yml b/data/distro_info.yml index 92300b3..45a90c3 100644 --- a/data/distro_info.yml +++ b/data/distro_info.yml @@ -6,9 +6,42 @@ ## or Debian series, but rather pointers to where that data lives: each dist ## entry below carries its series sources (the local distro-info CSV, with ## the network URL as fallback). +## +## Per-dist keys beyond the series pointers: +## mirrors: the archive mirrors, each a URL serving a set of +## architectures: `primary` (the main archive, whose url +## doubles as the dist's base URL) and, where they exist, +## the others (`ports`). `security_url` is the sibling +## host serving the -security pocket for the same arches +## (ports mirrors serve their own security); `archs` is +## an explicit list, or the `all` sentinel when one +## mirror serves every architecture (Debian's case — an +## exhaustive list would rot each time an arch is added). +## Host matching treats a URI as official when its host +## equals a mirror host or is a subdomain of it, so the +## country mirrors (fr.archive.ubuntu.com) count too. +## components: the archive components (main, universe, contrib, ...) +## a cross-build environment enables on official sources. +## Live archive operations keep resolving components from +## Release files; this is the offline default. +## cross_pockets: the pockets a cross-build environment enables for a +## series (`-updates`, ...). Deliberately not the +## `pockets` key: that one is the *search order* of pull, +## where backports must not fold in. +## build_profiles: the vendor's default DEB_BUILD_PROFILES (Ubuntu +## activates derivative.ubuntu noudeb, Debian none), +## mirroring what Dpkg::BuildProfiles resolves when the +## variable is unset. dist: debian: - base_url: https://deb.debian.org/debian + mirrors: + primary: + url: https://deb.debian.org/debian + # One mirror serves every architecture. + archs: all + components: [main, contrib, non-free, non-free-firmware] + cross_pockets: [updates, backports, security] + build_profiles: [] archive_keyring: https://ftp-master.debian.org/keys/archive-key-{series_num}.asc pockets: - updates @@ -82,7 +115,20 @@ dist: local: /usr/share/distro-info/debian.csv network: https://salsa.debian.org/debian/distro-info-data/-/raw/main/debian.csv ubuntu: - base_url: https://archive.ubuntu.com/ubuntu + mirrors: + primary: + url: https://archive.ubuntu.com/ubuntu + # Sibling host serving the -security pocket for the same arches. + security_url: http://security.ubuntu.com/ubuntu + archs: [amd64, i386] + ports: + # Everything else lives on the ports archive, which also serves + # its own -security pocket (no security_url needed). + url: http://ports.ubuntu.com/ubuntu-ports + archs: [armhf, arm64, ppc64el, riscv64, s390x] + components: [main, restricted, universe, multiverse] + cross_pockets: [updates, backports, security] + build_profiles: [derivative.ubuntu, noudeb] archive_keyring: https://archive.ubuntu.com/ubuntu/project/ubuntu-archive-keyring.gpg pockets: - updates diff --git a/src/distro_info.rs b/src/distro_info.rs index 402199f..76ef70a 100644 --- a/src/distro_info.rs +++ b/src/distro_info.rs @@ -2,6 +2,7 @@ use crate::data::embed_data; use chrono::NaiveDate; use lazy_static::lazy_static; use serde::Deserialize; +use std::collections::HashMap; use std::error::Error; use std::path::Path; use std::time::Duration; @@ -29,19 +30,69 @@ struct SeriesInfo { network: String, } +/// Architectures an archive mirror serves: an explicit list, or the `all` +/// sentinel meaning one mirror serves every architecture (Debian's mirror +/// setup — an exhaustive list would rot each time an arch is added) +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum MirrorArchs { + /// The `all` sentinel + All(String), + /// An explicit list of dpkg architecture names + List(Vec), +} + +impl MirrorArchs { + /// Whether the mirror serves `arch`. A scalar other than the `all` + /// sentinel serves nothing (a test locks that reading of the data). + fn serves(&self, arch: &str) -> bool { + match self { + MirrorArchs::All(sentinel) => sentinel == "all", + MirrorArchs::List(archs) => archs.iter().any(|a| a == arch), + } + } +} + +/// One archive mirror of a distribution: a URL serving a set of +/// architectures, plus the sibling host serving its `-security` pocket +/// for the same architectures (ports mirrors serve their own security) +#[derive(Debug, Deserialize)] +pub struct Mirror { + /// Base URL of the mirror (the primary mirror's URL doubles as the + /// dist's base URL, see [`get_base_url`]) + pub url: String, + /// Sibling host serving the `-security` pocket for these + /// architectures; `None` when the mirror serves its own security + #[serde(default)] + security_url: Option, + /// Architectures the mirror serves (see [`MirrorArchs`]) + archs: MirrorArchs, +} + +impl Mirror { + /// Whether the mirror serves `arch` + pub fn serves(&self, arch: &str) -> bool { + self.archs.serves(arch) + } +} + #[derive(Debug, Deserialize)] struct DistData { - base_url: String, + mirrors: HashMap, archive_keyring: String, pockets: Vec, #[serde(default)] sections: Vec, + components: Vec, + cross_pockets: Vec, + #[serde(default)] + build_profiles: Vec, series: SeriesInfo, } #[derive(Debug, Deserialize)] struct Data { - dist: std::collections::HashMap, + dist: HashMap, } embed_data! { @@ -220,6 +271,24 @@ pub fn supported_dists() -> Vec { dists } +/// Name of a dist's primary mirror entry: its URL doubles as the dist's +/// base URL ([`get_base_url`]) and is the first candidate of +/// [`mirror_for_arch`] +const PRIMARY_MIRROR: &str = "primary"; + +/// The data of a known distribution: the shared "unknown distribution" +/// error of the per-dist accessors +fn dist_data(dist: &str) -> Result<&'static DistData, Box> { + DATA.dist.get(dist).ok_or_else(|| { + format!( + "Unknown distribution '{}'. Supported distributions are: {}.", + dist, + supported_dists().join(", ") + ) + .into() + }) +} + /// Special changelog distribution marking an entry that has not been /// released to any archive series yet pub const UNRELEASED: &str = "UNRELEASED"; @@ -352,14 +421,7 @@ pub async fn get_dist_from_series(series: &str) -> Result /// /// Example: get_dist_pockets(ubuntu) => ["", "updates", "security", "proposed"] pub fn get_dist_pockets(dist: &str) -> Result, Box> { - let dist_data = DATA.dist.get(dist).ok_or_else(|| { - format!( - "Unknown distribution '{}'. Supported distributions are: {}.", - dist, - supported_dists().join(", ") - ) - })?; - let mut pockets = dist_data.pockets.clone(); + let mut pockets = dist_data(dist)?.pockets.clone(); // Explicitely add 'main' pocket, which is just the empty string, first pockets.insert(0, "".to_string()); @@ -367,18 +429,38 @@ pub fn get_dist_pockets(dist: &str) -> Result, Box> { Ok(pockets) } +/// Get the archive components of a distribution (ubuntu's main, +/// restricted, universe, multiverse; Debian's main, contrib, non-free, +/// non-free-firmware): the default set a build environment enables on its +/// official sources. Live archive operations keep resolving components +/// from Release files ([`get_components`]); this is the offline default. +pub fn get_dist_components(dist: &str) -> Result, Box> { + Ok(dist_data(dist)?.components.clone()) +} + +/// Get the pockets a cross-build environment enables for a series (the +/// `-` suite list is built from these): updates, +/// backports and security. Deliberately separate from +/// [`get_dist_pockets`], which is the *search order* of pull — folding +/// backports into it would change pull behavior. +pub fn get_cross_pockets(dist: &str) -> Result, Box> { + Ok(dist_data(dist)?.cross_pockets.clone()) +} + +/// Get the default build profiles of a distribution's vendor (Ubuntu +/// activates `derivative.ubuntu noudeb`, Debian none), mirroring what +/// `Dpkg::BuildProfiles` resolves when `DEB_BUILD_PROFILES` is unset. +/// Vendors are matched case-insensitively by the caller (dpkg's `Vendor:` +/// field keeps its original casing). +pub fn get_build_profiles(dist: &str) -> Result, Box> { + Ok(dist_data(dist)?.build_profiles.clone()) +} + /// Get the valid `Section` values of a distribution's packages, as accepted /// by its archives (a `section/subsection` in debian/control validates on /// the part before the '/') pub fn get_sections(dist: &str) -> Result, Box> { - let dist_data = DATA.dist.get(dist).ok_or_else(|| { - format!( - "Unknown distribution '{}'. Supported distributions are: {}.", - dist, - supported_dists().join(", ") - ) - })?; - Ok(dist_data.sections.clone()) + Ok(dist_data(dist)?.sections.clone()) } /// Get the sources URL for a distribution, series, pocket, and component @@ -391,23 +473,105 @@ pub fn get_sources_url(base_url: &str, series: &str, pocket: &str, component: &s format!("{base_url}/dists/{series}{pocket_full}/{component}/source/Sources.gz") } -/// Get the archive base URL for a distribution +/// Get the archive base URL for a distribution: the URL of its primary +/// mirror (the former `base_url` key folded into `mirrors.primary.url` +/// when the mirrors were modeled — the signature is kept so the pull +/// paths do not churn) /// /// Example: ubuntu => https://archive.ubuntu.com/ubuntu pub fn get_base_url(dist: &str) -> Result> { - DATA.dist - .get(dist) - .map(|d| d.base_url.clone()) + let mirror = dist_data(dist)? + .mirrors + .get(PRIMARY_MIRROR) .ok_or_else(|| { format!( - "Unknown distribution '{}'. Supported distributions are: {}.", - dist, + "Distribution '{dist}' has no '{PRIMARY_MIRROR}' mirror in the built-in \ + configuration. This is a bug; supported distributions are: {}.", + supported_dists().join(", ") + ) + })?; + Ok(mirror.url.clone()) +} + +/// The mirror of `dist` serving `arch`: the primary mirror first, then +/// the other mirrors by name, so the answer is deterministic. Debian's +/// `all` sentinel makes its primary mirror serve every architecture. +/// Errors when the dist is unknown or no mirror serves the architecture +/// (an architecture the built-in data does not know about). +/// +/// Example: mirror_for_arch(ubuntu, riscv64) => the ports mirror +pub fn mirror_for_arch(dist: &str, arch: &str) -> Result<&'static Mirror, Box> { + let data = dist_data(dist)?; + if let Some(primary) = data.mirrors.get(PRIMARY_MIRROR) + && primary.serves(arch) + { + return Ok(primary); + } + let mut others: Vec<&String> = data + .mirrors + .keys() + .filter(|name| name.as_str() != PRIMARY_MIRROR) + .collect(); + others.sort(); + others + .into_iter() + .filter_map(|name| data.mirrors.get(name)) + .find(|mirror| mirror.serves(arch)) + .ok_or_else(|| { + format!( + "No mirror of '{dist}' serves the '{arch}' architecture. Supported \ + distributions are: {}.", supported_dists().join(", ") ) .into() }) } +/// Host part of an apt-source URL: everything after the `://` scheme up +/// to the first `/` (a `:port` suffix stripped). URLs without a scheme +/// yield their leading segment. +fn url_host(url: &str) -> &str { + let after_scheme = url.split_once("://").map_or(url, |(_, rest)| rest); + let authority = after_scheme.split('/').next().unwrap_or(after_scheme); + authority + .split_once(':') + .map_or(authority, |(host, _)| host) +} + +/// Whether the host of `uri` is the host of `mirror_url` or a subdomain +/// of it: the old substring checks (`uri.contains("archive.ubuntu.com")`) +/// intentionally matched the country mirrors fronting each archive host +/// (`fr.archive.ubuntu.com`), and an exact host comparison would have +/// dropped them. The leading dot of the suffix keeps look-alike hosts +/// (`notarchive.ubuntu.com`) out. +fn uri_matches_url_host(uri: &str, mirror_url: &str) -> bool { + let host = url_host(mirror_url); + let uri_host = url_host(uri); + uri_host == host || uri_host.ends_with(&format!(".{host}")) +} + +/// Whether `uri` points at `mirror` or its security sibling: the URI's +/// host is the mirror's (or the sibling's) host or a subdomain of it (see +/// [`uri_matches_url_host`]) +pub fn is_mirror_source(mirror: &Mirror, uri: &str) -> bool { + uri_matches_url_host(uri, &mirror.url) + || mirror + .security_url + .as_deref() + .is_some_and(|security| uri_matches_url_host(uri, security)) +} + +/// Whether `uri` points at an official source of `dist` — one of its +/// archive mirrors or their security siblings — as opposed to a PPA or +/// another third-party repository. Unknown distributions match nothing. +pub fn is_official_source(dist: &str, uri: &str) -> bool { + DATA.dist.get(dist).is_some_and(|data| { + data.mirrors + .values() + .any(|mirror| is_mirror_source(mirror, uri)) + }) +} + /// Obtain the URLs for the archive keyrings of a distribution series /// /// For 'sid' and 'experimental', returns keyrings from the 3 latest releases @@ -565,6 +729,143 @@ mod tests { assert!(get_sections("not-a-distro").is_err()); } + /// The primary mirror's URL is the former `base_url`, byte for byte: + /// the pull paths build their archive URLs from it. + #[test] + fn test_primary_mirror_url_is_the_base_url() { + assert_eq!( + get_base_url("ubuntu").unwrap(), + "https://archive.ubuntu.com/ubuntu" + ); + assert_eq!( + get_base_url("debian").unwrap(), + "https://deb.debian.org/debian" + ); + assert!(get_base_url("not-a-distro").is_err()); + } + + /// Mirror-per-architecture resolution: the local architectures come + /// from Ubuntu's primary mirror, the others from ports; Debian's `all` + /// sentinel makes its one mirror serve everything, including + /// architectures the data never lists. + #[test] + fn test_mirror_for_arch() { + assert_eq!( + mirror_for_arch("ubuntu", "amd64").unwrap().url, + "https://archive.ubuntu.com/ubuntu" + ); + assert_eq!( + mirror_for_arch("ubuntu", "riscv64").unwrap().url, + "http://ports.ubuntu.com/ubuntu-ports" + ); + for arch in ["amd64", "riscv64", "brand-new"] { + assert_eq!( + mirror_for_arch("debian", arch).unwrap().url, + "https://deb.debian.org/debian", + "the `all` sentinel serves every architecture, including {arch}" + ); + } + // An architecture no Ubuntu mirror serves, and an unknown dist. + assert!(mirror_for_arch("ubuntu", "mips64el").is_err()); + assert!(mirror_for_arch("not-a-distro", "amd64").is_err()); + } + + /// The `archs` forms and their reading: the `all` sentinel serves + /// everything, an explicit list serves exactly its members, and any + /// other scalar serves nothing (a data bug a validation would have to + /// catch, hence the documented reading). + #[test] + fn test_mirror_archs_forms() { + let all: MirrorArchs = serde_yaml::from_str("all").unwrap(); + assert!(all.serves("anything")); + let list: MirrorArchs = serde_yaml::from_str("[amd64, i386]").unwrap(); + assert!(list.serves("amd64")); + assert!(!list.serves("arm64")); + let typo: MirrorArchs = serde_yaml::from_str("every").unwrap(); + assert!(!typo.serves("amd64")); + } + + /// Official-source matching is host-based but keeps matching the + /// country mirrors the old substring checks matched (`fr.archive. + /// ubuntu.com`): equality or a `.{host}` suffix, never a bare + /// substring — `notarchive.ubuntu.com` must not match. + #[test] + fn test_is_official_source_matches_country_mirrors_only() { + for uri in [ + "https://archive.ubuntu.com/ubuntu", + "http://security.ubuntu.com/ubuntu", + "http://ports.ubuntu.com/ubuntu-ports", + // Country mirrors front the same archives. + "http://fr.archive.ubuntu.com/ubuntu", + "https://de.security.ubuntu.com/ubuntu", + ] { + assert!(is_official_source("ubuntu", uri), "{uri}"); + } + for uri in [ + "https://deb.debian.org/debian", + "https://ppa.launchpadcontent.net/user/ppa/ubuntu", + "http://notarchive.ubuntu.com/ubuntu", + "http://archive.ubuntu.com.evil.example/ubuntu", + ] { + assert!(!is_official_source("ubuntu", uri), "{uri}"); + } + // Debian: its own mirror matches, Ubuntu's mirrors do not, and an + // unknown dist matches nothing. + assert!(is_official_source( + "debian", + "https://deb.debian.org/debian" + )); + assert!(!is_official_source( + "debian", + "http://security.ubuntu.com/ubuntu" + )); + assert!(!is_official_source( + "not-a-distro", + "https://deb.debian.org/debian" + )); + } + + /// The dist-level defaults the build paths read: components, + /// cross-build pockets (deliberately not the pull search order) and + /// vendor build profiles. + #[test] + fn test_dist_components_cross_pockets_and_build_profiles() { + assert_eq!( + get_dist_components("ubuntu").unwrap(), + vec!["main", "restricted", "universe", "multiverse"] + ); + assert!( + get_dist_components("debian") + .unwrap() + .contains(&"non-free-firmware".to_string()) + ); + + for dist in ["debian", "ubuntu"] { + assert_eq!( + get_cross_pockets(dist).unwrap(), + vec!["updates", "backports", "security"] + ); + } + // Not the pull search order: no 'proposed', no empty main pocket. + let cross = get_cross_pockets("ubuntu").unwrap(); + assert!(!cross.contains(&"proposed".to_string())); + assert!(!cross.contains(&"".to_string())); + + assert_eq!( + get_build_profiles("ubuntu").unwrap(), + vec!["derivative.ubuntu", "noudeb"] + ); + assert!(get_build_profiles("debian").unwrap().is_empty()); + + for getter in [ + get_dist_components as fn(&str) -> Result, Box>, + get_cross_pockets as fn(&str) -> Result, Box>, + get_build_profiles as fn(&str) -> Result, Box>, + ] { + assert!(getter("not-a-distro").is_err()); + } + } + #[test] fn test_parse_series_csv_malformed_rows() { // A short row (missing 'codename') is skipped, a row with an invalid