From 2225c33ef28d0d61ec8f054dcc35eed6d88f15b8 Mon Sep 17 00:00:00 2001 From: Valentin Haudiquet Date: Sun, 4 Oct 2026 21:20:24 +0200 Subject: [PATCH] feat(download): verify content against registry and CDN digests Downloaded content was hashed nowhere: OCI layer digests were only used to build blob URLs and distro tarballs were trusted as served, so a truncated or tampered rootfs went straight into the cache and from there into execution. Hash sha256 and sha512 while streaming and verify before the atomic rename, so a bad download never enters the cache: - OCI architecture manifests requested by digest are checked against that digest, since the layer digests read from them are trusted next; layer blobs are checked against the digest the manifest names. - Direct distro tarballs are checked against the checksums published next to them (Alpine: per-file .sha256; Ubuntu: SHA256SUMS in the release directory). A missing or unfetchable checksum artifact degrades to an unverified download with a warning, mirroring the freshness-check policy; a mismatch always fails. - Digest algorithms other than sha256/sha512 cannot be checked: warn and accept. --- Cargo.lock | 72 ++++++++++ crates/ecr/Cargo.toml | 3 + crates/ecr/src/distro.rs | 35 +++++ crates/ecr/src/download.rs | 264 ++++++++++++++++++++++++++++++++++++- 4 files changed, 368 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9336359..dd048e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,6 +104,15 @@ version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -249,6 +258,15 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "938e716cb1ade5d6c8f959c13a7248b889c07491fc7e41167c3afe20f8f0de1e" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -258,6 +276,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dirs" version = "6.0.0" @@ -313,6 +351,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sha2", "tar", "tempfile", "tokio", @@ -497,6 +536,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1477,6 +1526,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "2.0.1" @@ -1785,6 +1845,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1855,6 +1921,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "walkdir" version = "2.5.0" diff --git a/crates/ecr/Cargo.toml b/crates/ecr/Cargo.toml index 6fe4a45..6ce0b09 100644 --- a/crates/ecr/Cargo.toml +++ b/crates/ecr/Cargo.toml @@ -21,6 +21,9 @@ flate2 = "1" xz2 = "0.1" zstd = "0.13" +# Download integrity: verify content against registry/CDN digests +sha2 = "0.10" + # Logging facade (the CLI installs the sink) log = "0.4" diff --git a/crates/ecr/src/distro.rs b/crates/ecr/src/distro.rs index 7816f7b..84c3f9c 100644 --- a/crates/ecr/src/distro.rs +++ b/crates/ecr/src/distro.rs @@ -164,6 +164,19 @@ pub fn resolve_distro_url(distro: &Distro, version: Option<&str>, arch: &str) -> } } +/// URL of the checksum artifact covering the tarball at `tarball_url`, when +/// the distro publishes one. Alpine ships a per-file `.sha256`; Ubuntu +/// publishes a `SHA256SUMS` manifest per release directory. +pub fn checksum_artifact_url(distro: &Distro, tarball_url: &str) -> Option { + match distro { + Distro::Alpine => Some(format!("{}.sha256", tarball_url)), + Distro::Ubuntu => { + let dir = tarball_url.rsplit_once('/')?.0; + Some(format!("{}/SHA256SUMS", dir)) + } + } +} + /// Fetch the latest Ubuntu codename from a changelogs.ubuntu.com meta-release file. /// Pass the LTS-only URL to get the latest LTS, or the full URL for the latest release. fn fetch_ubuntu_codename(meta_release_url: &str) -> Result { @@ -412,6 +425,28 @@ fn resolve_alpine_url(version: Option<&str>, arch: &str) -> Result { mod tests { use super::*; + #[test] + fn checksum_artifact_url_per_distro() { + assert_eq!( + checksum_artifact_url( + &Distro::Alpine, + "https://dl-cdn.alpinelinux.org/alpine/v3.23/releases/x86_64/alpine-minirootfs-3.23.0-x86_64.tar.gz" + ) + .as_deref(), + Some( + "https://dl-cdn.alpinelinux.org/alpine/v3.23/releases/x86_64/alpine-minirootfs-3.23.0-x86_64.tar.gz.sha256" + ) + ); + assert_eq!( + checksum_artifact_url( + &Distro::Ubuntu, + "https://cdimage.ubuntu.com/ubuntu-base/noble/daily/current/noble-base-amd64.tar.gz" + ) + .as_deref(), + Some("https://cdimage.ubuntu.com/ubuntu-base/noble/daily/current/SHA256SUMS") + ); + } + fn oci(registry: &str, repository: &str, tag: &str) -> ImageSource { ImageSource::OciImage { registry: registry.to_string(), diff --git a/crates/ecr/src/download.rs b/crates/ecr/src/download.rs index 0bea6d9..e1208a1 100644 --- a/crates/ecr/src/download.rs +++ b/crates/ecr/src/download.rs @@ -3,10 +3,11 @@ use anyhow::{anyhow, Context, Result}; use futures_util::StreamExt; use reqwest::Client; use serde_json::Value; +use sha2::{Digest, Sha256, Sha512}; use std::path::{Path, PathBuf}; use std::time::Duration; -use crate::distro::ImageSource; +use crate::distro::{Distro, ImageSource}; /// Path of the digest sidecar file for a cached OCI image. /// e.g. `foo.tar.gz` → `foo.tar.gz.digest` @@ -16,6 +17,69 @@ pub fn digest_sidecar(cache_path: &Path) -> PathBuf { PathBuf::from(s) } +/// Incremental sha256+sha512 hasher used to verify downloaded content against +/// content-addressed digests. Both algorithms run side by side so the caller +/// can check whatever digest the manifest or checksum file names. +struct DownloadHasher { + sha256: Sha256, + sha512: Sha512, +} + +impl DownloadHasher { + fn new() -> Self { + DownloadHasher { + sha256: Sha256::new(), + sha512: Sha512::new(), + } + } + + fn update(&mut self, chunk: &[u8]) { + self.sha256.update(chunk); + self.sha512.update(chunk); + } + + /// Verify the streamed content. `expected` is an OCI `algo:hex` digest or + /// a bare sha256 hex string (distro checksum files); None verifies nothing. + /// Algorithms outside sha256/sha512 cannot be checked: warn and accept + /// rather than reject images a registry may legitimately serve. + fn verify(&self, expected: Option<&str>) -> Result<()> { + let Some(expected) = expected else { + return Ok(()); + }; + let (algo, hex) = match expected.split_once(':') { + Some((algo, hex)) => (algo, hex), + None => ("sha256", expected), + }; + + let actual = match algo { + "sha256" => hex_encode(&self.sha256.clone().finalize()), + "sha512" => hex_encode(&self.sha512.clone().finalize()), + other => { + log::warn!( + "digest algorithm '{}' not supported; skipping verification", + other + ); + return Ok(()); + } + }; + + if actual.eq_ignore_ascii_case(hex) { + Ok(()) + } else { + Err(anyhow!( + "content verification failed: expected {} {}, got {}", + algo, + hex, + actual + )) + } + } +} + +fn hex_encode(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{:02x}", b)).collect() +} + /// Fetch the current manifest digest for an OCI tag without downloading any /// layers. The registry returns the content-addressable digest in the /// `Docker-Content-Digest` response header of any manifest GET. @@ -91,7 +155,27 @@ pub fn download_image( if !url.starts_with("http://") && !url.starts_with("https://") { return Err(anyhow!("Unsupported URL scheme: {}", url)); } - rt.block_on(download_file_async(&url, dest, progress)) + // Verify the tarball against the checksum the distro publishes + // next to it. A fetch failure (missing artifact, network error) + // degrades to an unverified download with a warning rather than + // hard-failing, mirroring the digest freshness check in rootfs. + let expected = match rt.block_on(fetch_published_checksum(distro, &url)) { + Ok(expected) => Some(expected), + Err(e) => { + log::warn!( + "could not fetch published checksum ({}); \ + proceeding without verification", + e + ); + None + } + }; + rt.block_on(download_file_async( + &url, + dest, + progress, + expected.as_deref(), + )) } ImageSource::OciImage { registry, @@ -112,7 +196,63 @@ pub fn download_image( } } -async fn download_file_async(url: &str, dest: &Path, progress: &dyn Progress) -> Result<()> { +/// Fetch the sha256 the distro publishes for the tarball at `tarball_url`. +async fn fetch_published_checksum(distro: &Distro, tarball_url: &str) -> Result { + let Some(checksum_url) = crate::distro::checksum_artifact_url(distro, tarball_url) else { + return Err(anyhow!("no checksum artifact for {}", tarball_url)); + }; + + let client = Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .context("Failed to create HTTP client")?; + + let response = client + .get(&checksum_url) + .send() + .await + .with_context(|| format!("Failed to fetch checksum artifact {}", checksum_url))?; + + if !response.status().is_success() { + return Err(anyhow!( + "checksum artifact {} returned HTTP {}", + checksum_url, + response.status() + )); + } + + let text = response.text().await?; + let filename = tarball_url.rsplit('/').next().unwrap_or(""); + expected_digest_from_checksums(&text, filename).ok_or_else(|| { + anyhow!( + "checksum artifact {} does not list {}", + checksum_url, + filename + ) + }) +} + +/// Extract the digest for `filename` from a sha256sum-style checksum file +/// (lines of ` ` or ` *`). +fn expected_digest_from_checksums(text: &str, filename: &str) -> Option { + for line in text.lines() { + let mut fields = line.split_whitespace(); + let (Some(digest), Some(name)) = (fields.next(), fields.next()) else { + continue; + }; + if name.trim_start_matches('*') == filename { + return Some(digest.to_lowercase()); + } + } + None +} + +async fn download_file_async( + url: &str, + dest: &Path, + progress: &dyn Progress, + expected: Option<&str>, +) -> Result<()> { log::debug!("Downloading: {}", url); let client = Client::builder() @@ -148,8 +288,10 @@ async fn download_file_async(url: &str, dest: &Path, progress: &dyn Progress) -> let temp_dest = dest.with_extension("partial"); // Download into the temp file, then atomically rename to the final path. - // On any failure after the temp file is created we remove it so stale - // .partial files don't accumulate in the cache directory. + // Verification runs before the rename so a corrupted download never + // reaches the cache; on any failure after the temp file is created we + // remove it so stale .partial files don't accumulate. + let mut hasher = DownloadHasher::new(); let result: Result<()> = async { let mut file = tokio::fs::File::create(&temp_dest) .await @@ -159,10 +301,15 @@ async fn download_file_async(url: &str, dest: &Path, progress: &dyn Progress) -> while let Some(chunk) = stream.next().await { let chunk = chunk.context("Download interrupted")?; tokio::io::copy(&mut &chunk[..], &mut file).await?; + hasher.update(&chunk); progress.advance(chunk.len() as u64); } progress.finish(); + hasher.verify(expected)?; + if expected.is_some() { + log::debug!("Verified content digest: {}", dest.display()); + } tokio::fs::rename(&temp_dest, dest) .await @@ -321,6 +468,15 @@ async fn try_download_oci_image( } let arch_body = arch_response.text().await?; + + // The arch manifest was requested by digest: verify the response + // matches, since the layer digests read from it are trusted next. + let mut manifest_hasher = DownloadHasher::new(); + manifest_hasher.update(arch_body.as_bytes()); + manifest_hasher + .verify(Some(manifest_digest)) + .context("architecture manifest failed digest verification")?; + let arch_manifest: Value = serde_json::from_str(&arch_body) .context("Failed to parse architecture manifest JSON")?; @@ -378,7 +534,16 @@ async fn try_download_oci_image( let layer_path = temp_dir.path().join(&layer_name); let label = format!("Layer {}/{} ({})", i + 1, layers.len(), layer_name); - download_blob_with_auth(client, &blob_url, &layer_path, &token, &label, progress).await?; + download_blob_with_auth( + client, + &blob_url, + &layer_path, + &token, + &label, + layer_digest, + progress, + ) + .await?; layer_names.push(layer_name); } @@ -561,6 +726,7 @@ async fn download_blob_with_auth( dest: &Path, token: &str, label: &str, + expected_digest: &str, progress: &dyn Progress, ) -> Result<()> { let mut request = client.get(url); @@ -585,6 +751,7 @@ async fn download_blob_with_auth( .await .context("Failed to create destination file")?; + let mut hasher = DownloadHasher::new(); let mut stream = response.bytes_stream(); use futures_util::StreamExt; use tokio::io::AsyncWriteExt; @@ -594,12 +761,23 @@ async fn download_blob_with_auth( file.write_all(&chunk) .await .context("Failed to write chunk")?; + hasher.update(&chunk); progress.advance(chunk.len() as u64); } file.flush().await?; progress.finish(); + // The blob was addressed by this digest in the manifest; a mismatch means + // truncated or tampered content. The blob lives in a temp directory that + // is discarded on error, so a bad layer never reaches the cache. + hasher.verify(Some(expected_digest)).with_context(|| { + format!( + "layer blob {} failed verification against manifest digest", + url + ) + })?; + Ok(()) } @@ -626,7 +804,81 @@ fn media_type_to_extension(media_type: &str) -> &'static str { #[cfg(test)] mod tests { + use super::expected_digest_from_checksums; use super::parse_www_authenticate; + use super::DownloadHasher; + + // sha256("abc") and sha512("abc") — standard test vectors + const SHA256_ABC: &str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; + const SHA512_ABC: &str = "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a\ + 2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f"; + + fn hasher_with(content: &[u8]) -> DownloadHasher { + let mut hasher = DownloadHasher::new(); + hasher.update(content); + hasher + } + + #[test] + fn verify_oci_digests() { + let h = hasher_with(b"abc"); + assert!(h.verify(Some(&format!("sha256:{}", SHA256_ABC))).is_ok()); + assert!(h.verify(Some(&format!("sha512:{}", SHA512_ABC))).is_ok()); + } + + #[test] + fn verify_bare_hex_is_sha256() { + // distro checksum files carry a bare sha256 hex string + let h = hasher_with(b"abc"); + assert!(h.verify(Some(SHA256_ABC)).is_ok()); + } + + #[test] + fn verify_mismatch_errors() { + let h = hasher_with(b"abc"); + let wrong = format!("sha256:{}", "0".repeat(64)); + assert!(h.verify(Some(&wrong)).is_err()); + // right hash, wrong algorithm name pinned to it + let algo_mismatch = format!("sha512:{}", SHA256_ABC); + assert!(h.verify(Some(&algo_mismatch)).is_err()); + } + + #[test] + fn verify_none_and_unknown_algo_accept() { + let h = hasher_with(b"abc"); + assert!(h.verify(None).is_ok()); + assert!(h.verify(Some("md5:0123456789abcdef")).is_ok()); + } + + #[test] + fn checksum_parsing_bsd_style() { + // Alpine's per-file .sha256: " " + let text = format!("{} alpine-minirootfs-3.23.0-x86_64.tar.gz\n", SHA256_ABC); + assert_eq!( + expected_digest_from_checksums(&text, "alpine-minirootfs-3.23.0-x86_64.tar.gz"), + Some(SHA256_ABC.to_string()) + ); + } + + #[test] + fn checksum_parsing_binary_marker() { + // Ubuntu's SHA256SUMS: " *" (binary-mode marker) + let text = format!("{} *noble-base-amd64.tar.gz\n", SHA256_ABC); + assert_eq!( + expected_digest_from_checksums(&text, "noble-base-amd64.tar.gz"), + Some(SHA256_ABC.to_string()) + ); + } + + #[test] + fn checksum_parsing_missing_and_malformed() { + let text = format!("{} *other-file.tar.gz\nnot-a-checksum-line\n", SHA256_ABC); + assert_eq!( + expected_digest_from_checksums(&text, "noble-base-amd64.tar.gz"), + None + ); + assert_eq!(expected_digest_from_checksums("", "x"), None); + } #[test] fn test_docker_hub_challenge() {