feat(download): verify content against registry and CDN digests

Downloaded content was hashed nowhere: OCI layer digests were only
used to build blob URLs and distro tarballs were trusted as served,
so a truncated or tampered rootfs went straight into the cache and
from there into execution.

Hash sha256 and sha512 while streaming and verify before the atomic
rename, so a bad download never enters the cache:

- OCI architecture manifests requested by digest are checked against
  that digest, since the layer digests read from them are trusted
  next; layer blobs are checked against the digest the manifest names.
- Direct distro tarballs are checked against the checksums published
  next to them (Alpine: per-file .sha256; Ubuntu: SHA256SUMS in the
  release directory).  A missing or unfetchable checksum artifact
  degrades to an unverified download with a warning, mirroring the
  freshness-check policy; a mismatch always fails.
- Digest algorithms other than sha256/sha512 cannot be checked: warn
  and accept.
This commit is contained in:
2026-10-04 21:20:24 +02:00
parent da76455089
commit 2225c33ef2
4 changed files with 368 additions and 6 deletions
Generated
+72
View File
@@ -104,6 +104,15 @@ version = "2.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
[[package]]
name = "block-buffer"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array",
]
[[package]] [[package]]
name = "bumpalo" name = "bumpalo"
version = "3.20.3" version = "3.20.3"
@@ -249,6 +258,15 @@ version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "938e716cb1ade5d6c8f959c13a7248b889c07491fc7e41167c3afe20f8f0de1e" checksum = "938e716cb1ade5d6c8f959c13a7248b889c07491fc7e41167c3afe20f8f0de1e"
[[package]]
name = "cpufeatures"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "crc32fast" name = "crc32fast"
version = "1.5.0" version = "1.5.0"
@@ -258,6 +276,26 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "crypto-common"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
]
[[package]] [[package]]
name = "dirs" name = "dirs"
version = "6.0.0" version = "6.0.0"
@@ -313,6 +351,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"serde_yaml", "serde_yaml",
"sha2",
"tar", "tar",
"tempfile", "tempfile",
"tokio", "tokio",
@@ -497,6 +536,16 @@ dependencies = [
"slab", "slab",
] ]
[[package]]
name = "generic-array"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
]
[[package]] [[package]]
name = "getrandom" name = "getrandom"
version = "0.2.17" version = "0.2.17"
@@ -1477,6 +1526,17 @@ dependencies = [
"unsafe-libyaml", "unsafe-libyaml",
] ]
[[package]]
name = "sha2"
version = "0.10.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]] [[package]]
name = "shlex" name = "shlex"
version = "2.0.1" version = "2.0.1"
@@ -1785,6 +1845,12 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "typenum"
version = "1.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
[[package]] [[package]]
name = "unicode-ident" name = "unicode-ident"
version = "1.0.24" version = "1.0.24"
@@ -1855,6 +1921,12 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "version_check"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]] [[package]]
name = "walkdir" name = "walkdir"
version = "2.5.0" version = "2.5.0"
+3
View File
@@ -21,6 +21,9 @@ flate2 = "1"
xz2 = "0.1" xz2 = "0.1"
zstd = "0.13" zstd = "0.13"
# Download integrity: verify content against registry/CDN digests
sha2 = "0.10"
# Logging facade (the CLI installs the sink) # Logging facade (the CLI installs the sink)
log = "0.4" log = "0.4"
+35
View File
@@ -164,6 +164,19 @@ pub fn resolve_distro_url(distro: &Distro, version: Option<&str>, arch: &str) ->
} }
} }
/// URL of the checksum artifact covering the tarball at `tarball_url`, when
/// the distro publishes one. Alpine ships a per-file `.sha256`; Ubuntu
/// publishes a `SHA256SUMS` manifest per release directory.
pub fn checksum_artifact_url(distro: &Distro, tarball_url: &str) -> Option<String> {
match distro {
Distro::Alpine => Some(format!("{}.sha256", tarball_url)),
Distro::Ubuntu => {
let dir = tarball_url.rsplit_once('/')?.0;
Some(format!("{}/SHA256SUMS", dir))
}
}
}
/// Fetch the latest Ubuntu codename from a changelogs.ubuntu.com meta-release file. /// Fetch the latest Ubuntu codename from a changelogs.ubuntu.com meta-release file.
/// Pass the LTS-only URL to get the latest LTS, or the full URL for the latest release. /// Pass the LTS-only URL to get the latest LTS, or the full URL for the latest release.
fn fetch_ubuntu_codename(meta_release_url: &str) -> Result<String> { fn fetch_ubuntu_codename(meta_release_url: &str) -> Result<String> {
@@ -412,6 +425,28 @@ fn resolve_alpine_url(version: Option<&str>, arch: &str) -> Result<String> {
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn checksum_artifact_url_per_distro() {
assert_eq!(
checksum_artifact_url(
&Distro::Alpine,
"https://dl-cdn.alpinelinux.org/alpine/v3.23/releases/x86_64/alpine-minirootfs-3.23.0-x86_64.tar.gz"
)
.as_deref(),
Some(
"https://dl-cdn.alpinelinux.org/alpine/v3.23/releases/x86_64/alpine-minirootfs-3.23.0-x86_64.tar.gz.sha256"
)
);
assert_eq!(
checksum_artifact_url(
&Distro::Ubuntu,
"https://cdimage.ubuntu.com/ubuntu-base/noble/daily/current/noble-base-amd64.tar.gz"
)
.as_deref(),
Some("https://cdimage.ubuntu.com/ubuntu-base/noble/daily/current/SHA256SUMS")
);
}
fn oci(registry: &str, repository: &str, tag: &str) -> ImageSource { fn oci(registry: &str, repository: &str, tag: &str) -> ImageSource {
ImageSource::OciImage { ImageSource::OciImage {
registry: registry.to_string(), registry: registry.to_string(),
+258 -6
View File
@@ -3,10 +3,11 @@ use anyhow::{anyhow, Context, Result};
use futures_util::StreamExt; use futures_util::StreamExt;
use reqwest::Client; use reqwest::Client;
use serde_json::Value; use serde_json::Value;
use sha2::{Digest, Sha256, Sha512};
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::time::Duration; use std::time::Duration;
use crate::distro::ImageSource; use crate::distro::{Distro, ImageSource};
/// Path of the digest sidecar file for a cached OCI image. /// Path of the digest sidecar file for a cached OCI image.
/// e.g. `foo.tar.gz` → `foo.tar.gz.digest` /// e.g. `foo.tar.gz` → `foo.tar.gz.digest`
@@ -16,6 +17,69 @@ pub fn digest_sidecar(cache_path: &Path) -> PathBuf {
PathBuf::from(s) PathBuf::from(s)
} }
/// Incremental sha256+sha512 hasher used to verify downloaded content against
/// content-addressed digests. Both algorithms run side by side so the caller
/// can check whatever digest the manifest or checksum file names.
struct DownloadHasher {
sha256: Sha256,
sha512: Sha512,
}
impl DownloadHasher {
fn new() -> Self {
DownloadHasher {
sha256: Sha256::new(),
sha512: Sha512::new(),
}
}
fn update(&mut self, chunk: &[u8]) {
self.sha256.update(chunk);
self.sha512.update(chunk);
}
/// Verify the streamed content. `expected` is an OCI `algo:hex` digest or
/// a bare sha256 hex string (distro checksum files); None verifies nothing.
/// Algorithms outside sha256/sha512 cannot be checked: warn and accept
/// rather than reject images a registry may legitimately serve.
fn verify(&self, expected: Option<&str>) -> Result<()> {
let Some(expected) = expected else {
return Ok(());
};
let (algo, hex) = match expected.split_once(':') {
Some((algo, hex)) => (algo, hex),
None => ("sha256", expected),
};
let actual = match algo {
"sha256" => hex_encode(&self.sha256.clone().finalize()),
"sha512" => hex_encode(&self.sha512.clone().finalize()),
other => {
log::warn!(
"digest algorithm '{}' not supported; skipping verification",
other
);
return Ok(());
}
};
if actual.eq_ignore_ascii_case(hex) {
Ok(())
} else {
Err(anyhow!(
"content verification failed: expected {} {}, got {}",
algo,
hex,
actual
))
}
}
}
fn hex_encode(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{:02x}", b)).collect()
}
/// Fetch the current manifest digest for an OCI tag without downloading any /// Fetch the current manifest digest for an OCI tag without downloading any
/// layers. The registry returns the content-addressable digest in the /// layers. The registry returns the content-addressable digest in the
/// `Docker-Content-Digest` response header of any manifest GET. /// `Docker-Content-Digest` response header of any manifest GET.
@@ -91,7 +155,27 @@ pub fn download_image(
if !url.starts_with("http://") && !url.starts_with("https://") { if !url.starts_with("http://") && !url.starts_with("https://") {
return Err(anyhow!("Unsupported URL scheme: {}", url)); return Err(anyhow!("Unsupported URL scheme: {}", url));
} }
rt.block_on(download_file_async(&url, dest, progress)) // Verify the tarball against the checksum the distro publishes
// next to it. A fetch failure (missing artifact, network error)
// degrades to an unverified download with a warning rather than
// hard-failing, mirroring the digest freshness check in rootfs.
let expected = match rt.block_on(fetch_published_checksum(distro, &url)) {
Ok(expected) => Some(expected),
Err(e) => {
log::warn!(
"could not fetch published checksum ({}); \
proceeding without verification",
e
);
None
}
};
rt.block_on(download_file_async(
&url,
dest,
progress,
expected.as_deref(),
))
} }
ImageSource::OciImage { ImageSource::OciImage {
registry, registry,
@@ -112,7 +196,63 @@ pub fn download_image(
} }
} }
async fn download_file_async(url: &str, dest: &Path, progress: &dyn Progress) -> Result<()> { /// Fetch the sha256 the distro publishes for the tarball at `tarball_url`.
async fn fetch_published_checksum(distro: &Distro, tarball_url: &str) -> Result<String> {
let Some(checksum_url) = crate::distro::checksum_artifact_url(distro, tarball_url) else {
return Err(anyhow!("no checksum artifact for {}", tarball_url));
};
let client = Client::builder()
.timeout(Duration::from_secs(30))
.build()
.context("Failed to create HTTP client")?;
let response = client
.get(&checksum_url)
.send()
.await
.with_context(|| format!("Failed to fetch checksum artifact {}", checksum_url))?;
if !response.status().is_success() {
return Err(anyhow!(
"checksum artifact {} returned HTTP {}",
checksum_url,
response.status()
));
}
let text = response.text().await?;
let filename = tarball_url.rsplit('/').next().unwrap_or("");
expected_digest_from_checksums(&text, filename).ok_or_else(|| {
anyhow!(
"checksum artifact {} does not list {}",
checksum_url,
filename
)
})
}
/// Extract the digest for `filename` from a sha256sum-style checksum file
/// (lines of `<hex> <name>` or `<hex> *<name>`).
fn expected_digest_from_checksums(text: &str, filename: &str) -> Option<String> {
for line in text.lines() {
let mut fields = line.split_whitespace();
let (Some(digest), Some(name)) = (fields.next(), fields.next()) else {
continue;
};
if name.trim_start_matches('*') == filename {
return Some(digest.to_lowercase());
}
}
None
}
async fn download_file_async(
url: &str,
dest: &Path,
progress: &dyn Progress,
expected: Option<&str>,
) -> Result<()> {
log::debug!("Downloading: {}", url); log::debug!("Downloading: {}", url);
let client = Client::builder() let client = Client::builder()
@@ -148,8 +288,10 @@ async fn download_file_async(url: &str, dest: &Path, progress: &dyn Progress) ->
let temp_dest = dest.with_extension("partial"); let temp_dest = dest.with_extension("partial");
// Download into the temp file, then atomically rename to the final path. // Download into the temp file, then atomically rename to the final path.
// On any failure after the temp file is created we remove it so stale // Verification runs before the rename so a corrupted download never
// .partial files don't accumulate in the cache directory. // reaches the cache; on any failure after the temp file is created we
// remove it so stale .partial files don't accumulate.
let mut hasher = DownloadHasher::new();
let result: Result<()> = async { let result: Result<()> = async {
let mut file = tokio::fs::File::create(&temp_dest) let mut file = tokio::fs::File::create(&temp_dest)
.await .await
@@ -159,10 +301,15 @@ async fn download_file_async(url: &str, dest: &Path, progress: &dyn Progress) ->
while let Some(chunk) = stream.next().await { while let Some(chunk) = stream.next().await {
let chunk = chunk.context("Download interrupted")?; let chunk = chunk.context("Download interrupted")?;
tokio::io::copy(&mut &chunk[..], &mut file).await?; tokio::io::copy(&mut &chunk[..], &mut file).await?;
hasher.update(&chunk);
progress.advance(chunk.len() as u64); progress.advance(chunk.len() as u64);
} }
progress.finish(); progress.finish();
hasher.verify(expected)?;
if expected.is_some() {
log::debug!("Verified content digest: {}", dest.display());
}
tokio::fs::rename(&temp_dest, dest) tokio::fs::rename(&temp_dest, dest)
.await .await
@@ -321,6 +468,15 @@ async fn try_download_oci_image(
} }
let arch_body = arch_response.text().await?; let arch_body = arch_response.text().await?;
// The arch manifest was requested by digest: verify the response
// matches, since the layer digests read from it are trusted next.
let mut manifest_hasher = DownloadHasher::new();
manifest_hasher.update(arch_body.as_bytes());
manifest_hasher
.verify(Some(manifest_digest))
.context("architecture manifest failed digest verification")?;
let arch_manifest: Value = serde_json::from_str(&arch_body) let arch_manifest: Value = serde_json::from_str(&arch_body)
.context("Failed to parse architecture manifest JSON")?; .context("Failed to parse architecture manifest JSON")?;
@@ -378,7 +534,16 @@ async fn try_download_oci_image(
let layer_path = temp_dir.path().join(&layer_name); let layer_path = temp_dir.path().join(&layer_name);
let label = format!("Layer {}/{} ({})", i + 1, layers.len(), layer_name); let label = format!("Layer {}/{} ({})", i + 1, layers.len(), layer_name);
download_blob_with_auth(client, &blob_url, &layer_path, &token, &label, progress).await?; download_blob_with_auth(
client,
&blob_url,
&layer_path,
&token,
&label,
layer_digest,
progress,
)
.await?;
layer_names.push(layer_name); layer_names.push(layer_name);
} }
@@ -561,6 +726,7 @@ async fn download_blob_with_auth(
dest: &Path, dest: &Path,
token: &str, token: &str,
label: &str, label: &str,
expected_digest: &str,
progress: &dyn Progress, progress: &dyn Progress,
) -> Result<()> { ) -> Result<()> {
let mut request = client.get(url); let mut request = client.get(url);
@@ -585,6 +751,7 @@ async fn download_blob_with_auth(
.await .await
.context("Failed to create destination file")?; .context("Failed to create destination file")?;
let mut hasher = DownloadHasher::new();
let mut stream = response.bytes_stream(); let mut stream = response.bytes_stream();
use futures_util::StreamExt; use futures_util::StreamExt;
use tokio::io::AsyncWriteExt; use tokio::io::AsyncWriteExt;
@@ -594,12 +761,23 @@ async fn download_blob_with_auth(
file.write_all(&chunk) file.write_all(&chunk)
.await .await
.context("Failed to write chunk")?; .context("Failed to write chunk")?;
hasher.update(&chunk);
progress.advance(chunk.len() as u64); progress.advance(chunk.len() as u64);
} }
file.flush().await?; file.flush().await?;
progress.finish(); progress.finish();
// The blob was addressed by this digest in the manifest; a mismatch means
// truncated or tampered content. The blob lives in a temp directory that
// is discarded on error, so a bad layer never reaches the cache.
hasher.verify(Some(expected_digest)).with_context(|| {
format!(
"layer blob {} failed verification against manifest digest",
url
)
})?;
Ok(()) Ok(())
} }
@@ -626,7 +804,81 @@ fn media_type_to_extension(media_type: &str) -> &'static str {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::expected_digest_from_checksums;
use super::parse_www_authenticate; use super::parse_www_authenticate;
use super::DownloadHasher;
// sha256("abc") and sha512("abc") — standard test vectors
const SHA256_ABC: &str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad";
const SHA512_ABC: &str = "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a\
2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f";
fn hasher_with(content: &[u8]) -> DownloadHasher {
let mut hasher = DownloadHasher::new();
hasher.update(content);
hasher
}
#[test]
fn verify_oci_digests() {
let h = hasher_with(b"abc");
assert!(h.verify(Some(&format!("sha256:{}", SHA256_ABC))).is_ok());
assert!(h.verify(Some(&format!("sha512:{}", SHA512_ABC))).is_ok());
}
#[test]
fn verify_bare_hex_is_sha256() {
// distro checksum files carry a bare sha256 hex string
let h = hasher_with(b"abc");
assert!(h.verify(Some(SHA256_ABC)).is_ok());
}
#[test]
fn verify_mismatch_errors() {
let h = hasher_with(b"abc");
let wrong = format!("sha256:{}", "0".repeat(64));
assert!(h.verify(Some(&wrong)).is_err());
// right hash, wrong algorithm name pinned to it
let algo_mismatch = format!("sha512:{}", SHA256_ABC);
assert!(h.verify(Some(&algo_mismatch)).is_err());
}
#[test]
fn verify_none_and_unknown_algo_accept() {
let h = hasher_with(b"abc");
assert!(h.verify(None).is_ok());
assert!(h.verify(Some("md5:0123456789abcdef")).is_ok());
}
#[test]
fn checksum_parsing_bsd_style() {
// Alpine's per-file .sha256: "<hex> <filename>"
let text = format!("{} alpine-minirootfs-3.23.0-x86_64.tar.gz\n", SHA256_ABC);
assert_eq!(
expected_digest_from_checksums(&text, "alpine-minirootfs-3.23.0-x86_64.tar.gz"),
Some(SHA256_ABC.to_string())
);
}
#[test]
fn checksum_parsing_binary_marker() {
// Ubuntu's SHA256SUMS: "<hex> *<filename>" (binary-mode marker)
let text = format!("{} *noble-base-amd64.tar.gz\n", SHA256_ABC);
assert_eq!(
expected_digest_from_checksums(&text, "noble-base-amd64.tar.gz"),
Some(SHA256_ABC.to_string())
);
}
#[test]
fn checksum_parsing_missing_and_malformed() {
let text = format!("{} *other-file.tar.gz\nnot-a-checksum-line\n", SHA256_ABC);
assert_eq!(
expected_digest_from_checksums(&text, "noble-base-amd64.tar.gz"),
None
);
assert_eq!(expected_digest_from_checksums("", "x"), None);
}
#[test] #[test]
fn test_docker_hub_challenge() { fn test_docker_hub_challenge() {