distro_info: model distro mirrors, components and build profiles in the yaml data

data/distro_info.yml

@@ -6,9 +6,42 @@
 ## or Debian series, but rather pointers to where that data lives: each dist
 ## entry below carries its series sources (the local distro-info CSV, with
 ## the network URL as fallback).
+##
+## Per-dist keys beyond the series pointers:
+##   mirrors:        the archive mirrors, each a URL serving a set of
+##                   architectures: `primary` (the main archive, whose url
+##                   doubles as the dist's base URL) and, where they exist,
+##                   the others (`ports`). `security_url` is the sibling
+##                   host serving the -security pocket for the same arches
+##                   (ports mirrors serve their own security); `archs` is
+##                   an explicit list, or the `all` sentinel when one
+##                   mirror serves every architecture (Debian's case — an
+##                   exhaustive list would rot each time an arch is added).
+##                   Host matching treats a URI as official when its host
+##                   equals a mirror host or is a subdomain of it, so the
+##                   country mirrors (fr.archive.ubuntu.com) count too.
+##   components:     the archive components (main, universe, contrib, ...)
+##                   a cross-build environment enables on official sources.
+##                   Live archive operations keep resolving components from
+##                   Release files; this is the offline default.
+##   cross_pockets:  the pockets a cross-build environment enables for a
+##                   series (`<series>-updates`, ...). Deliberately not the
+##                   `pockets` key: that one is the *search order* of pull,
+##                   where backports must not fold in.
+##   build_profiles: the vendor's default DEB_BUILD_PROFILES (Ubuntu
+##                   activates derivative.ubuntu noudeb, Debian none),
+##                   mirroring what Dpkg::BuildProfiles resolves when the
+##                   variable is unset.
 dist:
   debian:
-    base_url: https://deb.debian.org/debian
+    mirrors:
+      primary:
+        url: https://deb.debian.org/debian
+        # One mirror serves every architecture.
+        archs: all
+    components: [main, contrib, non-free, non-free-firmware]
+    cross_pockets: [updates, backports, security]
+    build_profiles: []
     archive_keyring: https://ftp-master.debian.org/keys/archive-key-{series_num}.asc
     pockets:
       - updates
@@ -82,7 +115,20 @@ dist:
       local: /usr/share/distro-info/debian.csv
       network: https://salsa.debian.org/debian/distro-info-data/-/raw/main/debian.csv
   ubuntu:
-    base_url: https://archive.ubuntu.com/ubuntu
+    mirrors:
+      primary:
+        url: https://archive.ubuntu.com/ubuntu
+        # Sibling host serving the -security pocket for the same arches.
+        security_url: http://security.ubuntu.com/ubuntu
+        archs: [amd64, i386]
+      ports:
+        # Everything else lives on the ports archive, which also serves
+        # its own -security pocket (no security_url needed).
+        url: http://ports.ubuntu.com/ubuntu-ports
+        archs: [armhf, arm64, ppc64el, riscv64, s390x]
+    components: [main, restricted, universe, multiverse]
+    cross_pockets: [updates, backports, security]
+    build_profiles: [derivative.ubuntu, noudeb]
     archive_keyring: https://archive.ubuntu.com/ubuntu/project/ubuntu-archive-keyring.gpg
     pockets:
       - updates

src/distro_info.rs

@@ -2,6 +2,7 @@ use crate::data::embed_data;
 use chrono::NaiveDate;
 use lazy_static::lazy_static;
 use serde::Deserialize;
+use std::collections::HashMap;
 use std::error::Error;
 use std::path::Path;
 use std::time::Duration;
@@ -29,19 +30,69 @@ struct SeriesInfo {
     network: String,
 }
 
+/// Architectures an archive mirror serves: an explicit list, or the `all`
+/// sentinel meaning one mirror serves every architecture (Debian's mirror
+/// setup — an exhaustive list would rot each time an arch is added)
+#[derive(Debug, Deserialize)]
+#[serde(untagged)]
+enum MirrorArchs {
+    /// The `all` sentinel
+    All(String),
+    /// An explicit list of dpkg architecture names
+    List(Vec<String>),
+}
+
+impl MirrorArchs {
+    /// Whether the mirror serves `arch`. A scalar other than the `all`
+    /// sentinel serves nothing (a test locks that reading of the data).
+    fn serves(&self, arch: &str) -> bool {
+        match self {
+            MirrorArchs::All(sentinel) => sentinel == "all",
+            MirrorArchs::List(archs) => archs.iter().any(|a| a == arch),
+        }
+    }
+}
+
+/// One archive mirror of a distribution: a URL serving a set of
+/// architectures, plus the sibling host serving its `-security` pocket
+/// for the same architectures (ports mirrors serve their own security)
+#[derive(Debug, Deserialize)]
+pub struct Mirror {
+    /// Base URL of the mirror (the primary mirror's URL doubles as the
+    /// dist's base URL, see [`get_base_url`])
+    pub url: String,
+    /// Sibling host serving the `-security` pocket for these
+    /// architectures; `None` when the mirror serves its own security
+    #[serde(default)]
+    security_url: Option<String>,
+    /// Architectures the mirror serves (see [`MirrorArchs`])
+    archs: MirrorArchs,
+}
+
+impl Mirror {
+    /// Whether the mirror serves `arch`
+    pub fn serves(&self, arch: &str) -> bool {
+        self.archs.serves(arch)
+    }
+}
+
 #[derive(Debug, Deserialize)]
 struct DistData {
-    base_url: String,
+    mirrors: HashMap<String, Mirror>,
     archive_keyring: String,
     pockets: Vec<String>,
     #[serde(default)]
     sections: Vec<String>,
+    components: Vec<String>,
+    cross_pockets: Vec<String>,
+    #[serde(default)]
+    build_profiles: Vec<String>,
     series: SeriesInfo,
 }
 
 #[derive(Debug, Deserialize)]
 struct Data {
-    dist: std::collections::HashMap<String, DistData>,
+    dist: HashMap<String, DistData>,
 }
 
 embed_data! {
@@ -220,6 +271,24 @@ pub fn supported_dists() -> Vec<String> {
     dists
 }
 
+/// Name of a dist's primary mirror entry: its URL doubles as the dist's
+/// base URL ([`get_base_url`]) and is the first candidate of
+/// [`mirror_for_arch`]
+const PRIMARY_MIRROR: &str = "primary";
+
+/// The data of a known distribution: the shared "unknown distribution"
+/// error of the per-dist accessors
+fn dist_data(dist: &str) -> Result<&'static DistData, Box<dyn Error>> {
+    DATA.dist.get(dist).ok_or_else(|| {
+        format!(
+            "Unknown distribution '{}'. Supported distributions are: {}.",
+            dist,
+            supported_dists().join(", ")
+        )
+        .into()
+    })
+}
+
 /// Special changelog distribution marking an entry that has not been
 /// released to any archive series yet
 pub const UNRELEASED: &str = "UNRELEASED";
@@ -352,14 +421,7 @@ pub async fn get_dist_from_series(series: &str) -> Result<String, Box<dyn Error>
 ///
 /// Example: get_dist_pockets(ubuntu) => ["", "updates", "security", "proposed"]
 pub fn get_dist_pockets(dist: &str) -> Result<Vec<String>, Box<dyn Error>> {
-    let dist_data = DATA.dist.get(dist).ok_or_else(|| {
+    let mut pockets = dist_data(dist)?.pockets.clone();
-        format!(
-            "Unknown distribution '{}'. Supported distributions are: {}.",
-            dist,
-            supported_dists().join(", ")
-        )
-    })?;
-    let mut pockets = dist_data.pockets.clone();
 
     // Explicitely add 'main' pocket, which is just the empty string, first
     pockets.insert(0, "".to_string());
@@ -367,18 +429,38 @@ pub fn get_dist_pockets(dist: &str) -> Result<Vec<String>, Box<dyn Error>> {
     Ok(pockets)
 }
 
+/// Get the archive components of a distribution (ubuntu's main,
+/// restricted, universe, multiverse; Debian's main, contrib, non-free,
+/// non-free-firmware): the default set a build environment enables on its
+/// official sources. Live archive operations keep resolving components
+/// from Release files ([`get_components`]); this is the offline default.
+pub fn get_dist_components(dist: &str) -> Result<Vec<String>, Box<dyn Error>> {
+    Ok(dist_data(dist)?.components.clone())
+}
+
+/// Get the pockets a cross-build environment enables for a series (the
+/// `<series>-<pocket>` suite list is built from these): updates,
+/// backports and security. Deliberately separate from
+/// [`get_dist_pockets`], which is the *search order* of pull — folding
+/// backports into it would change pull behavior.
+pub fn get_cross_pockets(dist: &str) -> Result<Vec<String>, Box<dyn Error>> {
+    Ok(dist_data(dist)?.cross_pockets.clone())
+}
+
+/// Get the default build profiles of a distribution's vendor (Ubuntu
+/// activates `derivative.ubuntu noudeb`, Debian none), mirroring what
+/// `Dpkg::BuildProfiles` resolves when `DEB_BUILD_PROFILES` is unset.
+/// Vendors are matched case-insensitively by the caller (dpkg's `Vendor:`
+/// field keeps its original casing).
+pub fn get_build_profiles(dist: &str) -> Result<Vec<String>, Box<dyn Error>> {
+    Ok(dist_data(dist)?.build_profiles.clone())
+}
+
 /// Get the valid `Section` values of a distribution's packages, as accepted
 /// by its archives (a `section/subsection` in debian/control validates on
 /// the part before the '/')
 pub fn get_sections(dist: &str) -> Result<Vec<String>, Box<dyn Error>> {
-    let dist_data = DATA.dist.get(dist).ok_or_else(|| {
+    Ok(dist_data(dist)?.sections.clone())
-        format!(
-            "Unknown distribution '{}'. Supported distributions are: {}.",
-            dist,
-            supported_dists().join(", ")
-        )
-    })?;
-    Ok(dist_data.sections.clone())
 }
 
 /// Get the sources URL for a distribution, series, pocket, and component
@@ -391,23 +473,105 @@ pub fn get_sources_url(base_url: &str, series: &str, pocket: &str, component: &s
     format!("{base_url}/dists/{series}{pocket_full}/{component}/source/Sources.gz")
 }
 
-/// Get the archive base URL for a distribution
+/// Get the archive base URL for a distribution: the URL of its primary
+/// mirror (the former `base_url` key folded into `mirrors.primary.url`
+/// when the mirrors were modeled — the signature is kept so the pull
+/// paths do not churn)
 ///
 /// Example: ubuntu => https://archive.ubuntu.com/ubuntu
 pub fn get_base_url(dist: &str) -> Result<String, Box<dyn Error>> {
-    DATA.dist
+    let mirror = dist_data(dist)?
-        .get(dist)
+        .mirrors
-        .map(|d| d.base_url.clone())
+        .get(PRIMARY_MIRROR)
         .ok_or_else(|| {
             format!(
-                "Unknown distribution '{}'. Supported distributions are: {}.",
+                "Distribution '{dist}' has no '{PRIMARY_MIRROR}' mirror in the built-in \
-                dist,
+             configuration. This is a bug; supported distributions are: {}.",
+                supported_dists().join(", ")
+            )
+        })?;
+    Ok(mirror.url.clone())
+}
+
+/// The mirror of `dist` serving `arch`: the primary mirror first, then
+/// the other mirrors by name, so the answer is deterministic. Debian's
+/// `all` sentinel makes its primary mirror serve every architecture.
+/// Errors when the dist is unknown or no mirror serves the architecture
+/// (an architecture the built-in data does not know about).
+///
+/// Example: mirror_for_arch(ubuntu, riscv64) => the ports mirror
+pub fn mirror_for_arch(dist: &str, arch: &str) -> Result<&'static Mirror, Box<dyn Error>> {
+    let data = dist_data(dist)?;
+    if let Some(primary) = data.mirrors.get(PRIMARY_MIRROR)
+        && primary.serves(arch)
+    {
+        return Ok(primary);
+    }
+    let mut others: Vec<&String> = data
+        .mirrors
+        .keys()
+        .filter(|name| name.as_str() != PRIMARY_MIRROR)
+        .collect();
+    others.sort();
+    others
+        .into_iter()
+        .filter_map(|name| data.mirrors.get(name))
+        .find(|mirror| mirror.serves(arch))
+        .ok_or_else(|| {
+            format!(
+                "No mirror of '{dist}' serves the '{arch}' architecture. Supported \
+                 distributions are: {}.",
                 supported_dists().join(", ")
             )
             .into()
         })
 }
 
+/// Host part of an apt-source URL: everything after the `://` scheme up
+/// to the first `/` (a `:port` suffix stripped). URLs without a scheme
+/// yield their leading segment.
+fn url_host(url: &str) -> &str {
+    let after_scheme = url.split_once("://").map_or(url, |(_, rest)| rest);
+    let authority = after_scheme.split('/').next().unwrap_or(after_scheme);
+    authority
+        .split_once(':')
+        .map_or(authority, |(host, _)| host)
+}
+
+/// Whether the host of `uri` is the host of `mirror_url` or a subdomain
+/// of it: the old substring checks (`uri.contains("archive.ubuntu.com")`)
+/// intentionally matched the country mirrors fronting each archive host
+/// (`fr.archive.ubuntu.com`), and an exact host comparison would have
+/// dropped them. The leading dot of the suffix keeps look-alike hosts
+/// (`notarchive.ubuntu.com`) out.
+fn uri_matches_url_host(uri: &str, mirror_url: &str) -> bool {
+    let host = url_host(mirror_url);
+    let uri_host = url_host(uri);
+    uri_host == host || uri_host.ends_with(&format!(".{host}"))
+}
+
+/// Whether `uri` points at `mirror` or its security sibling: the URI's
+/// host is the mirror's (or the sibling's) host or a subdomain of it (see
+/// [`uri_matches_url_host`])
+pub fn is_mirror_source(mirror: &Mirror, uri: &str) -> bool {
+    uri_matches_url_host(uri, &mirror.url)
+        || mirror
+            .security_url
+            .as_deref()
+            .is_some_and(|security| uri_matches_url_host(uri, security))
+}
+
+/// Whether `uri` points at an official source of `dist` — one of its
+/// archive mirrors or their security siblings — as opposed to a PPA or
+/// another third-party repository. Unknown distributions match nothing.
+pub fn is_official_source(dist: &str, uri: &str) -> bool {
+    DATA.dist.get(dist).is_some_and(|data| {
+        data.mirrors
+            .values()
+            .any(|mirror| is_mirror_source(mirror, uri))
+    })
+}
+
 /// Obtain the URLs for the archive keyrings of a distribution series
 ///
 /// For 'sid' and 'experimental', returns keyrings from the 3 latest releases
@@ -565,6 +729,143 @@ mod tests {
         assert!(get_sections("not-a-distro").is_err());
     }
 
+    /// The primary mirror's URL is the former `base_url`, byte for byte:
+    /// the pull paths build their archive URLs from it.
+    #[test]
+    fn test_primary_mirror_url_is_the_base_url() {
+        assert_eq!(
+            get_base_url("ubuntu").unwrap(),
+            "https://archive.ubuntu.com/ubuntu"
+        );
+        assert_eq!(
+            get_base_url("debian").unwrap(),
+            "https://deb.debian.org/debian"
+        );
+        assert!(get_base_url("not-a-distro").is_err());
+    }
+
+    /// Mirror-per-architecture resolution: the local architectures come
+    /// from Ubuntu's primary mirror, the others from ports; Debian's `all`
+    /// sentinel makes its one mirror serve everything, including
+    /// architectures the data never lists.
+    #[test]
+    fn test_mirror_for_arch() {
+        assert_eq!(
+            mirror_for_arch("ubuntu", "amd64").unwrap().url,
+            "https://archive.ubuntu.com/ubuntu"
+        );
+        assert_eq!(
+            mirror_for_arch("ubuntu", "riscv64").unwrap().url,
+            "http://ports.ubuntu.com/ubuntu-ports"
+        );
+        for arch in ["amd64", "riscv64", "brand-new"] {
+            assert_eq!(
+                mirror_for_arch("debian", arch).unwrap().url,
+                "https://deb.debian.org/debian",
+                "the `all` sentinel serves every architecture, including {arch}"
+            );
+        }
+        // An architecture no Ubuntu mirror serves, and an unknown dist.
+        assert!(mirror_for_arch("ubuntu", "mips64el").is_err());
+        assert!(mirror_for_arch("not-a-distro", "amd64").is_err());
+    }
+
+    /// The `archs` forms and their reading: the `all` sentinel serves
+    /// everything, an explicit list serves exactly its members, and any
+    /// other scalar serves nothing (a data bug a validation would have to
+    /// catch, hence the documented reading).
+    #[test]
+    fn test_mirror_archs_forms() {
+        let all: MirrorArchs = serde_yaml::from_str("all").unwrap();
+        assert!(all.serves("anything"));
+        let list: MirrorArchs = serde_yaml::from_str("[amd64, i386]").unwrap();
+        assert!(list.serves("amd64"));
+        assert!(!list.serves("arm64"));
+        let typo: MirrorArchs = serde_yaml::from_str("every").unwrap();
+        assert!(!typo.serves("amd64"));
+    }
+
+    /// Official-source matching is host-based but keeps matching the
+    /// country mirrors the old substring checks matched (`fr.archive.
+    /// ubuntu.com`): equality or a `.{host}` suffix, never a bare
+    /// substring — `notarchive.ubuntu.com` must not match.
+    #[test]
+    fn test_is_official_source_matches_country_mirrors_only() {
+        for uri in [
+            "https://archive.ubuntu.com/ubuntu",
+            "http://security.ubuntu.com/ubuntu",
+            "http://ports.ubuntu.com/ubuntu-ports",
+            // Country mirrors front the same archives.
+            "http://fr.archive.ubuntu.com/ubuntu",
+            "https://de.security.ubuntu.com/ubuntu",
+        ] {
+            assert!(is_official_source("ubuntu", uri), "{uri}");
+        }
+        for uri in [
+            "https://deb.debian.org/debian",
+            "https://ppa.launchpadcontent.net/user/ppa/ubuntu",
+            "http://notarchive.ubuntu.com/ubuntu",
+            "http://archive.ubuntu.com.evil.example/ubuntu",
+        ] {
+            assert!(!is_official_source("ubuntu", uri), "{uri}");
+        }
+        // Debian: its own mirror matches, Ubuntu's mirrors do not, and an
+        // unknown dist matches nothing.
+        assert!(is_official_source(
+            "debian",
+            "https://deb.debian.org/debian"
+        ));
+        assert!(!is_official_source(
+            "debian",
+            "http://security.ubuntu.com/ubuntu"
+        ));
+        assert!(!is_official_source(
+            "not-a-distro",
+            "https://deb.debian.org/debian"
+        ));
+    }
+
+    /// The dist-level defaults the build paths read: components,
+    /// cross-build pockets (deliberately not the pull search order) and
+    /// vendor build profiles.
+    #[test]
+    fn test_dist_components_cross_pockets_and_build_profiles() {
+        assert_eq!(
+            get_dist_components("ubuntu").unwrap(),
+            vec!["main", "restricted", "universe", "multiverse"]
+        );
+        assert!(
+            get_dist_components("debian")
+                .unwrap()
+                .contains(&"non-free-firmware".to_string())
+        );
+
+        for dist in ["debian", "ubuntu"] {
+            assert_eq!(
+                get_cross_pockets(dist).unwrap(),
+                vec!["updates", "backports", "security"]
+            );
+        }
+        // Not the pull search order: no 'proposed', no empty main pocket.
+        let cross = get_cross_pockets("ubuntu").unwrap();
+        assert!(!cross.contains(&"proposed".to_string()));
+        assert!(!cross.contains(&"".to_string()));
+
+        assert_eq!(
+            get_build_profiles("ubuntu").unwrap(),
+            vec!["derivative.ubuntu", "noudeb"]
+        );
+        assert!(get_build_profiles("debian").unwrap().is_empty());
+
+        for getter in [
+            get_dist_components as fn(&str) -> Result<Vec<String>, Box<dyn Error>>,
+            get_cross_pockets as fn(&str) -> Result<Vec<String>, Box<dyn Error>>,
+            get_build_profiles as fn(&str) -> Result<Vec<String>, Box<dyn Error>>,
+        ] {
+            assert!(getter("not-a-distro").is_err());
+        }
+    }
+
     #[test]
     fn test_parse_series_csv_malformed_rows() {
         // A short row (missing 'codename') is skipped, a row with an invalid