feat(distro): accept digest-pinned OCI references

Docker pulls ubuntu@sha256:... and repo:tag@sha256:...; ecrou
errored on the '@'. parse_oci_ref now peels the pin (which contains
the reference's only legal post-slash colon), validates it as
algo:hex, and the manifest is fetched by digest - the tag, when
co-present, is ignored the way docker ignores it. Pinned
references are immutable by definition: the :latest freshness
check skips them, and the cache filename carries the pin so a
tagged ref and its pin never share an entry. Pull parameters
gather into registry::OciPull.
This commit is contained in:
vhaudiquet committed 2026-10-30 00:08:27 +01:00
1 parent 200aac1f89
commit 28f24abb6f
5 files changed
+217 -43

No files matched your search

+9 -7
View File
@@ -299,10 +299,12 @@ ecrou probe
``` ```
The image is a distro name (`alpine`, `ubuntu:noble`), an OCI The image is a distro name (`alpine`, `ubuntu:noble`), an OCI
reference (`ghcr.io/org/image:tag`), or a local rootfs archive named reference (`ghcr.io/org/image:tag`, `ubuntu@sha256:…`,
with a path (`./rootfs.tar.gz`, `/tmp/rootfs.tar`): a path reference is `repo:tag@sha256:…` — the digest wins over a co-present tag, as in
read in place on every run, never downloaded and never cached as a docker), or a local rootfs archive named with a path
pristine entry. (`./rootfs.tar.gz`, `/tmp/rootfs.tar`): a path reference is read in
place on every run, never downloaded and never cached as a pristine
entry.
| Flag | Default | Description | | Flag | Default | Description |
|------|---------|-------------| |------|---------|-------------|
@@ -394,9 +396,9 @@ stop:
hard-failing. That is a deliberate trade, not an oversight. hard-failing. That is a deliberate trade, not an oversight.
- The cache trusts its own entries. Once written, an entry is never - The cache trusts its own entries. Once written, an entry is never
re-verified, except OCI `:latest` tags, which are re-checked against re-verified, except OCI `:latest` tags, which are re-checked against
the registry digest on every run. Pinned tags and distro tarballs are the registry digest on every run. Digest pins, pinned tags and
assumed immutable; `--no-cache` and the digest-changed path are the distro tarballs are assumed immutable; `--no-cache` and the
escape hatches. digest-changed path are the escape hatches.
- Concurrent runs of the same image are safe, not efficient: each - Concurrent runs of the same image are safe, not efficient: each
writes its own partial file and the last atomic rename wins, so two writes its own partial file and the last atomic rename wins, so two
racers download twice rather than corrupt anything. racers download twice rather than corrupt anything.
+125 -8
View File
@@ -25,6 +25,10 @@ pub enum ImageSource {
registry: String, registry: String,
repository: String, repository: String,
tag: String, tag: String,
/// A digest pin (`sha256:…`), when the reference carries one
/// (`repo:tag@sha256:…` or `repo@sha256:…`): the manifest is
/// fetched by digest and the image is immutable by definition.
digest: Option<String>,
architecture: String, architecture: String,
}, },
/// A rootfs archive on the host filesystem, named with a path /// A rootfs archive on the host filesystem, named with a path
@@ -49,6 +53,7 @@ impl Distro {
/// - Distro with version: ubuntu:noble, alpine:3.19 /// - Distro with version: ubuntu:noble, alpine:3.19
/// - OCI image references: docker://ubuntu:latest, quay.io/centos/centos:stream9 /// - OCI image references: docker://ubuntu:latest, quay.io/centos/centos:stream9
/// - Docker Hub shorthand: ubuntu:latest (without registry prefix) /// - Docker Hub shorthand: ubuntu:latest (without registry prefix)
/// - Digest pins, as docker accepts them: ubuntu@sha256:…, repo:tag@sha256:…
pub fn parse_image_ref(input: &str, arch: &str) -> Result<ImageSource> { pub fn parse_image_ref(input: &str, arch: &str) -> Result<ImageSource> {
let input = input.trim(); let input = input.trim();
@@ -94,6 +99,7 @@ pub fn parse_image_ref(input: &str, arch: &str) -> Result<ImageSource> {
registry: "docker.io".to_string(), registry: "docker.io".to_string(),
repository: "library/archlinux".to_string(), repository: "library/archlinux".to_string(),
tag: version.unwrap_or_else(|| "latest".to_string()), tag: version.unwrap_or_else(|| "latest".to_string()),
digest: None,
architecture: oci_arch, architecture: oci_arch,
}) })
} }
@@ -104,6 +110,7 @@ pub fn parse_image_ref(input: &str, arch: &str) -> Result<ImageSource> {
registry: "docker.io".to_string(), registry: "docker.io".to_string(),
repository: "gentoo/stage3".to_string(), repository: "gentoo/stage3".to_string(),
tag: version.unwrap_or_else(|| "latest".to_string()), tag: version.unwrap_or_else(|| "latest".to_string()),
digest: None,
architecture: oci_arch, architecture: oci_arch,
}) })
} }
@@ -112,26 +119,37 @@ pub fn parse_image_ref(input: &str, arch: &str) -> Result<ImageSource> {
} }
} }
/// Parse OCI image reference (registry/repo:tag or repo:tag) /// Parse OCI image reference (registry/repo:tag[@digest] or repo:tag[@digest])
fn parse_oci_ref(input: &str, arch: &str) -> anyhow::Result<ImageSource> { fn parse_oci_ref(input: &str, arch: &str) -> anyhow::Result<ImageSource> {
// Split off a digest pin first: '@' after the last '/' introduces
// `algo:hex`, which itself contains a colon, so it must be peeled
// before tag splitting. Docker accepts `repo:tag@digest` (digest
// wins) and bare `repo@digest`; so do we.
let (without_digest, digest) = match digest_separator(input) {
Some(at) => (
&input[..at],
Some(validate_digest(&input[at + 1..]).map_err(anyhow::Error::msg)?),
),
None => (input, None),
};
// Split off the tag. A ':' is a tag separator only when it appears after // Split off the tag. A ':' is a tag separator only when it appears after
// the last '/'; any ':' before a '/' is a port number // the last '/'; any ':' before a '/' is a port number
// (e.g. localhost:5000/image). With no '/' the single ':' is the tag. // (e.g. localhost:5000/image). With no '/' the single ':' is the tag.
let (without_tag, tag) = if let Some(slash_pos) = input.rfind('/') { let (without_tag, tag) = if let Some(slash_pos) = without_digest.rfind('/') {
let after_slash = &input[slash_pos + 1..]; let after_slash = &without_digest[slash_pos + 1..];
if let Some(colon_pos) = after_slash.find(':') { if let Some(colon_pos) = after_slash.find(':') {
( (
&input[..slash_pos + 1 + colon_pos], &without_digest[..slash_pos + 1 + colon_pos],
input[slash_pos + 1 + colon_pos + 1..].to_string(), without_digest[slash_pos + 1 + colon_pos + 1..].to_string(),
) )
} else { } else {
(input, "latest".to_string()) (without_digest, "latest".to_string())
} }
} else { } else {
// No slash: bare "ubuntu" or "ubuntu:latest" // No slash: bare "ubuntu" or "ubuntu:latest"
match input.split_once(':') { match without_digest.split_once(':') {
Some((name, t)) => (name, t.to_string()), Some((name, t)) => (name, t.to_string()),
None => (input, "latest".to_string()), None => (without_digest, "latest".to_string()),
} }
}; };
@@ -157,10 +175,46 @@ fn parse_oci_ref(input: &str, arch: &str) -> anyhow::Result<ImageSource> {
registry, registry,
repository, repository,
tag, tag,
digest,
architecture: oci_arch, architecture: oci_arch,
}) })
} }
/// Byte offset of the digest separator ('@' after the last '/'), if the
/// reference carries a digest pin. '@' is not legal in registries,
/// repositories, or tags, so the first one after the last '/' is
/// unambiguous.
fn digest_separator(input: &str) -> Option<usize> {
let after_slash = match input.rfind('/') {
Some(slash) => &input[slash + 1..],
None => input,
};
after_slash
.find('@')
.map(|at| input.len() - after_slash.len() + at)
}
/// Validate a digest pin: `algo:hex`, hex at least 32 digits — sha256
/// and sha512 are the digests registries actually issue. Returns the
/// lowercased digest.
fn validate_digest(digest: &str) -> anyhow::Result<String> {
let (algo, hex) = digest
.split_once(':')
.ok_or_else(|| anyhow::anyhow!("invalid image digest {digest:?}: expected algo:hex"))?;
let ok = !algo.is_empty()
&& algo
.chars()
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit())
&& hex.len() >= 32
&& hex.chars().all(|c| c.is_ascii_hexdigit());
if !ok {
return Err(anyhow::anyhow!(
"invalid image digest {digest:?}: expected e.g. sha256:<64 hex digits>"
));
}
Ok(format!("{}:{}", algo, hex.to_lowercase()))
}
/// Map architecture to OCI standard names /// Map architecture to OCI standard names
pub fn map_oci_arch(arch: &str) -> String { pub fn map_oci_arch(arch: &str) -> String {
crate::utils::map_oci_arch(arch) crate::utils::map_oci_arch(arch)
@@ -480,6 +534,7 @@ mod tests {
registry: registry.to_string(), registry: registry.to_string(),
repository: repository.to_string(), repository: repository.to_string(),
tag: tag.to_string(), tag: tag.to_string(),
digest: None,
architecture: "amd64".to_string(), architecture: "amd64".to_string(),
} }
} }
@@ -575,6 +630,68 @@ mod tests {
); );
} }
const SHA256_ABC: &str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad";
#[test]
fn digest_pins_parse_like_docker_references() {
// Bare digest ref
let src = parse(&format!("ubuntu@sha256:{SHA256_ABC}"));
let ImageSource::OciImage {
registry,
repository,
tag,
digest,
..
} = src
else {
panic!("expected OCI source");
};
assert_eq!(registry, "docker.io");
assert_eq!(repository, "library/ubuntu");
assert_eq!(tag, "latest");
assert_eq!(
digest.as_deref(),
Some(format!("sha256:{SHA256_ABC}").as_str())
);
// Tag + digest: the digest wins, the tag rides along
let src = parse(&format!("ubuntu:24.04@sha256:{SHA256_ABC}"));
let ImageSource::OciImage { tag, digest, .. } = src else {
panic!("expected OCI source");
};
assert_eq!(tag, "24.04");
assert!(digest.is_some());
// Registry-qualified digest ref, port included
let src = parse(&format!("localhost:5000/org/img@sha256:{SHA256_ABC}"));
let ImageSource::OciImage {
registry,
repository,
digest,
..
} = src
else {
panic!("expected OCI source");
};
assert_eq!(registry, "localhost:5000");
assert_eq!(repository, "org/img");
assert!(digest.is_some());
}
#[test]
fn malformed_digests_are_named_errors() {
for bad in [
"ubuntu@sha256:tooshort",
"ubuntu@sha256",
"ubuntu@sha256:zz7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
] {
assert!(
parse_image_ref(&format!("docker://{bad}"), "amd64").is_err(),
"{bad}"
);
}
}
#[test] #[test]
fn path_like_registry_references_are_never_local_files() { fn path_like_registry_references_are_never_local_files() {
// A leading path component that is a hostname stays OCI: only // A leading path component that is a hostname stays OCI: only
+5 -3
View File
@@ -143,17 +143,19 @@ pub fn download_image(
registry, registry,
repository, repository,
tag, tag,
digest,
architecture, architecture,
} => { } => {
log::debug!("Pulling OCI image: {}/{}:{}", registry, repository, tag); log::debug!("Pulling OCI image: {}/{}:{}", registry, repository, tag);
crate::registry::download_oci_image( let pull = crate::registry::OciPull {
registry, registry,
repository, repository,
tag, tag,
digest: digest.as_deref(),
architecture, architecture,
dest, dest,
progress, };
)?; crate::registry::download_oci_image(&pull, progress)?;
Ok(()) Ok(())
} }
// A local archive is read in place by the prepare step; there // A local archive is read in place by the prepare step; there
+38 -22
View File
@@ -63,21 +63,26 @@ fn fetch_oci_digest_inner(registry: &str, repository: &str, tag: &str) -> anyhow
.ok_or_else(|| anyhow!("Registry did not return a Docker-Content-Digest header")) .ok_or_else(|| anyhow!("Registry did not return a Docker-Content-Digest header"))
} }
pub(crate) fn download_oci_image( /// What a pull needs: the parsed reference and where the bundle lands.
registry: &str, pub(crate) struct OciPull<'a> {
repository: &str, pub registry: &'a str,
tag: &str, pub repository: &'a str,
arch: &str, pub tag: &'a str,
dest: &Path, /// A digest pin: the manifest is fetched by digest and the tag,
progress: &dyn Progress, /// when present too, is ignored the way docker ignores it.
) -> anyhow::Result<()> { pub digest: Option<&'a str>,
pub architecture: &'a str,
pub dest: &'a Path,
}
pub(crate) fn download_oci_image(pull: &OciPull, progress: &dyn Progress) -> anyhow::Result<()> {
let client = http_client(None)?; let client = http_client(None)?;
let digest = try_download_oci_image(&client, registry, repository, tag, arch, dest, progress)?; let digest = try_download_oci_image(&client, pull, progress)?;
// Persist the manifest digest so the next run can skip the download when // Persist the manifest digest so the next run can skip the download when
// the tag hasn't changed (see fetch_oci_digest / digest_sidecar). // the tag hasn't changed (see fetch_oci_digest / digest_sidecar).
if !digest.is_empty() { if !digest.is_empty() {
let _ = std::fs::write(digest_sidecar(dest), &digest); let _ = std::fs::write(digest_sidecar(pull.dest), &digest);
} }
Ok(()) Ok(())
@@ -85,25 +90,35 @@ pub(crate) fn download_oci_image(
fn try_download_oci_image( fn try_download_oci_image(
client: &Client, client: &Client,
registry: &str, pull: &OciPull,
repository: &str,
tag: &str,
arch: &str,
dest: &Path,
progress: &dyn Progress, progress: &dyn Progress,
) -> anyhow::Result<String> { ) -> anyhow::Result<String> {
let OciPull {
registry,
repository,
tag,
digest: digest_pin,
architecture: arch,
dest,
} = *pull;
// Get authentication token for the registry // Get authentication token for the registry
let token = get_auth_token(client, registry, repository)?; let token = get_auth_token(client, registry, repository)?;
// Construct manifest URL based on registry // A digest pin fetches the manifest by digest — the pin IS the
// reference; the tag, when present too, is ignored the way docker
// ignores it. Otherwise the tag names the manifest.
let reference = digest_pin.unwrap_or(tag);
let manifest_url = if registry == "docker.io" { let manifest_url = if registry == "docker.io" {
// Docker Hub uses registry-1.docker.io for API // Docker Hub uses registry-1.docker.io for API
format!( format!(
"https://registry-1.docker.io/v2/{}/manifests/{}", "https://registry-1.docker.io/v2/{}/manifests/{}",
repository, tag repository, reference
) )
} else { } else {
format!("https://{}/v2/{}/manifests/{}", registry, repository, tag) format!(
"https://{}/v2/{}/manifests/{}",
registry, repository, reference
)
}; };
log::debug!("Fetching manifest from: {}", manifest_url); log::debug!("Fetching manifest from: {}", manifest_url);
@@ -123,10 +138,11 @@ fn try_download_oci_image(
)); ));
} }
// Capture the tag-level digest before consuming the response body. // Capture the reference-level digest before consuming the response
// This is the content-addressable identity of the manifest (or manifest // body. This is the content-addressable identity of the manifest (or
// list) at this tag — used to detect whether the tag has moved since the // manifest list) at this tag — used to detect whether the tag has
// last download. // moved since the last download. For a digest pin it is the pin
// itself, and the freshness machinery never consults it.
let tag_digest = response let tag_digest = response
.headers() .headers()
.get("Docker-Content-Digest") .get("Docker-Content-Digest")
+40 -3
View File
@@ -83,12 +83,14 @@ fn oci_freshness_changed(source: &ImageSource, cache_path: &Path) -> bool {
registry, registry,
repository, repository,
tag, tag,
digest,
.. ..
} = source } = source
else { else {
return false; return false;
}; };
if tag != "latest" { // A digest pin is immutable by definition; so is any non-latest tag.
if digest.is_some() || tag != "latest" {
return false; return false;
} }
match fetch_oci_digest(registry, repository, tag) { match fetch_oci_digest(registry, repository, tag) {
@@ -637,14 +639,18 @@ fn generate_cache_filename(source: &ImageSource, arch: &str) -> String {
registry, registry,
repository, repository,
tag, tag,
digest,
architecture, architecture,
} => { } => {
// Sanitize for filename // Sanitize for filename. A digest pin replaces the tag in
// the name: two references differing only by pin must not
// share an entry.
let safe_registry = registry.replace(['.', ':'], "_"); let safe_registry = registry.replace(['.', ':'], "_");
let safe_repo = repository.replace(['/', ':'], "_"); let safe_repo = repository.replace(['/', ':'], "_");
let reference = digest.as_deref().unwrap_or(tag).replace(':', "_");
format!( format!(
"oci-{}-{}-{}-{}.tar.gz", "oci-{}-{}-{}-{}.tar.gz",
safe_registry, safe_repo, tag, architecture safe_registry, safe_repo, reference, architecture
) )
} }
ImageSource::LocalTarball { path } => { ImageSource::LocalTarball { path } => {
@@ -736,6 +742,7 @@ mod tests {
registry: "docker.io".to_string(), registry: "docker.io".to_string(),
repository: "library/ubuntu".to_string(), repository: "library/ubuntu".to_string(),
tag: "latest".to_string(), tag: "latest".to_string(),
digest: None,
architecture: "amd64".to_string(), architecture: "amd64".to_string(),
}; };
assert_eq!( assert_eq!(
@@ -909,6 +916,36 @@ mod tests {
assert_ne!(empty_name, grown); assert_ne!(empty_name, grown);
} }
#[test]
fn a_digest_pin_names_its_own_cache_entry() {
let cache = RootfsCache::new(std::path::PathBuf::from("/cache"));
let tagged = crate::distro::parse_image_ref("ubuntu:24.04", "amd64").unwrap();
let pinned = crate::distro::parse_image_ref(
"ubuntu:24.04@sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
"amd64",
)
.unwrap();
assert_ne!(
cache.tarball_path(&tagged, "amd64"),
cache.tarball_path(&pinned, "amd64"),
"a pin must not share the tag's entry"
);
}
#[test]
fn a_digest_pin_is_never_checked_for_freshness() {
// Even with a :latest tag riding along, the pin wins: this
// must return without any registry contact (it would hang the
// test suite offline if it tried).
let pinned = crate::distro::parse_image_ref(
"ubuntu:latest@sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
"amd64",
)
.unwrap();
let existing = tempfile::NamedTempFile::new().unwrap();
assert!(!oci_freshness_changed(&pinned, existing.path()));
}
#[test] #[test]
fn a_layer_key_splices_into_the_entry_name() { fn a_layer_key_splices_into_the_entry_name() {
let cache = RootfsCache::new(std::path::PathBuf::from("/cache")); let cache = RootfsCache::new(std::path::PathBuf::from("/cache"));