feat(download): verify content against registry and CDN digests
Downloaded content was hashed nowhere: OCI layer digests were only used to build blob URLs and distro tarballs were trusted as served, so a truncated or tampered rootfs went straight into the cache and from there into execution. Hash sha256 and sha512 while streaming and verify before the atomic rename, so a bad download never enters the cache: - OCI architecture manifests requested by digest are checked against that digest, since the layer digests read from them are trusted next; layer blobs are checked against the digest the manifest names. - Direct distro tarballs are checked against the checksums published next to them (Alpine: per-file .sha256; Ubuntu: SHA256SUMS in the release directory). A missing or unfetchable checksum artifact degrades to an unverified download with a warning, mirroring the freshness-check policy; a mismatch always fails. - Digest algorithms other than sha256/sha512 cannot be checked: warn and accept.
This commit is contained in:
@@ -21,6 +21,9 @@ flate2 = "1"
|
||||
xz2 = "0.1"
|
||||
zstd = "0.13"
|
||||
|
||||
# Download integrity: verify content against registry/CDN digests
|
||||
sha2 = "0.10"
|
||||
|
||||
# Logging facade (the CLI installs the sink)
|
||||
log = "0.4"
|
||||
|
||||
|
||||
@@ -164,6 +164,19 @@ pub fn resolve_distro_url(distro: &Distro, version: Option<&str>, arch: &str) ->
|
||||
}
|
||||
}
|
||||
|
||||
/// URL of the checksum artifact covering the tarball at `tarball_url`, when
|
||||
/// the distro publishes one. Alpine ships a per-file `.sha256`; Ubuntu
|
||||
/// publishes a `SHA256SUMS` manifest per release directory.
|
||||
pub fn checksum_artifact_url(distro: &Distro, tarball_url: &str) -> Option<String> {
|
||||
match distro {
|
||||
Distro::Alpine => Some(format!("{}.sha256", tarball_url)),
|
||||
Distro::Ubuntu => {
|
||||
let dir = tarball_url.rsplit_once('/')?.0;
|
||||
Some(format!("{}/SHA256SUMS", dir))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch the latest Ubuntu codename from a changelogs.ubuntu.com meta-release file.
|
||||
/// Pass the LTS-only URL to get the latest LTS, or the full URL for the latest release.
|
||||
fn fetch_ubuntu_codename(meta_release_url: &str) -> Result<String> {
|
||||
@@ -412,6 +425,28 @@ fn resolve_alpine_url(version: Option<&str>, arch: &str) -> Result<String> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn checksum_artifact_url_per_distro() {
|
||||
assert_eq!(
|
||||
checksum_artifact_url(
|
||||
&Distro::Alpine,
|
||||
"https://dl-cdn.alpinelinux.org/alpine/v3.23/releases/x86_64/alpine-minirootfs-3.23.0-x86_64.tar.gz"
|
||||
)
|
||||
.as_deref(),
|
||||
Some(
|
||||
"https://dl-cdn.alpinelinux.org/alpine/v3.23/releases/x86_64/alpine-minirootfs-3.23.0-x86_64.tar.gz.sha256"
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
checksum_artifact_url(
|
||||
&Distro::Ubuntu,
|
||||
"https://cdimage.ubuntu.com/ubuntu-base/noble/daily/current/noble-base-amd64.tar.gz"
|
||||
)
|
||||
.as_deref(),
|
||||
Some("https://cdimage.ubuntu.com/ubuntu-base/noble/daily/current/SHA256SUMS")
|
||||
);
|
||||
}
|
||||
|
||||
fn oci(registry: &str, repository: &str, tag: &str) -> ImageSource {
|
||||
ImageSource::OciImage {
|
||||
registry: registry.to_string(),
|
||||
|
||||
+258
-6
@@ -3,10 +3,11 @@ use anyhow::{anyhow, Context, Result};
|
||||
use futures_util::StreamExt;
|
||||
use reqwest::Client;
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256, Sha512};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::distro::ImageSource;
|
||||
use crate::distro::{Distro, ImageSource};
|
||||
|
||||
/// Path of the digest sidecar file for a cached OCI image.
|
||||
/// e.g. `foo.tar.gz` → `foo.tar.gz.digest`
|
||||
@@ -16,6 +17,69 @@ pub fn digest_sidecar(cache_path: &Path) -> PathBuf {
|
||||
PathBuf::from(s)
|
||||
}
|
||||
|
||||
/// Incremental sha256+sha512 hasher used to verify downloaded content against
|
||||
/// content-addressed digests. Both algorithms run side by side so the caller
|
||||
/// can check whatever digest the manifest or checksum file names.
|
||||
struct DownloadHasher {
|
||||
sha256: Sha256,
|
||||
sha512: Sha512,
|
||||
}
|
||||
|
||||
impl DownloadHasher {
|
||||
fn new() -> Self {
|
||||
DownloadHasher {
|
||||
sha256: Sha256::new(),
|
||||
sha512: Sha512::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn update(&mut self, chunk: &[u8]) {
|
||||
self.sha256.update(chunk);
|
||||
self.sha512.update(chunk);
|
||||
}
|
||||
|
||||
/// Verify the streamed content. `expected` is an OCI `algo:hex` digest or
|
||||
/// a bare sha256 hex string (distro checksum files); None verifies nothing.
|
||||
/// Algorithms outside sha256/sha512 cannot be checked: warn and accept
|
||||
/// rather than reject images a registry may legitimately serve.
|
||||
fn verify(&self, expected: Option<&str>) -> Result<()> {
|
||||
let Some(expected) = expected else {
|
||||
return Ok(());
|
||||
};
|
||||
let (algo, hex) = match expected.split_once(':') {
|
||||
Some((algo, hex)) => (algo, hex),
|
||||
None => ("sha256", expected),
|
||||
};
|
||||
|
||||
let actual = match algo {
|
||||
"sha256" => hex_encode(&self.sha256.clone().finalize()),
|
||||
"sha512" => hex_encode(&self.sha512.clone().finalize()),
|
||||
other => {
|
||||
log::warn!(
|
||||
"digest algorithm '{}' not supported; skipping verification",
|
||||
other
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
if actual.eq_ignore_ascii_case(hex) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(anyhow!(
|
||||
"content verification failed: expected {} {}, got {}",
|
||||
algo,
|
||||
hex,
|
||||
actual
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn hex_encode(bytes: &[u8]) -> String {
|
||||
bytes.iter().map(|b| format!("{:02x}", b)).collect()
|
||||
}
|
||||
|
||||
/// Fetch the current manifest digest for an OCI tag without downloading any
|
||||
/// layers. The registry returns the content-addressable digest in the
|
||||
/// `Docker-Content-Digest` response header of any manifest GET.
|
||||
@@ -91,7 +155,27 @@ pub fn download_image(
|
||||
if !url.starts_with("http://") && !url.starts_with("https://") {
|
||||
return Err(anyhow!("Unsupported URL scheme: {}", url));
|
||||
}
|
||||
rt.block_on(download_file_async(&url, dest, progress))
|
||||
// Verify the tarball against the checksum the distro publishes
|
||||
// next to it. A fetch failure (missing artifact, network error)
|
||||
// degrades to an unverified download with a warning rather than
|
||||
// hard-failing, mirroring the digest freshness check in rootfs.
|
||||
let expected = match rt.block_on(fetch_published_checksum(distro, &url)) {
|
||||
Ok(expected) => Some(expected),
|
||||
Err(e) => {
|
||||
log::warn!(
|
||||
"could not fetch published checksum ({}); \
|
||||
proceeding without verification",
|
||||
e
|
||||
);
|
||||
None
|
||||
}
|
||||
};
|
||||
rt.block_on(download_file_async(
|
||||
&url,
|
||||
dest,
|
||||
progress,
|
||||
expected.as_deref(),
|
||||
))
|
||||
}
|
||||
ImageSource::OciImage {
|
||||
registry,
|
||||
@@ -112,7 +196,63 @@ pub fn download_image(
|
||||
}
|
||||
}
|
||||
|
||||
async fn download_file_async(url: &str, dest: &Path, progress: &dyn Progress) -> Result<()> {
|
||||
/// Fetch the sha256 the distro publishes for the tarball at `tarball_url`.
|
||||
async fn fetch_published_checksum(distro: &Distro, tarball_url: &str) -> Result<String> {
|
||||
let Some(checksum_url) = crate::distro::checksum_artifact_url(distro, tarball_url) else {
|
||||
return Err(anyhow!("no checksum artifact for {}", tarball_url));
|
||||
};
|
||||
|
||||
let client = Client::builder()
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.context("Failed to create HTTP client")?;
|
||||
|
||||
let response = client
|
||||
.get(&checksum_url)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("Failed to fetch checksum artifact {}", checksum_url))?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(anyhow!(
|
||||
"checksum artifact {} returned HTTP {}",
|
||||
checksum_url,
|
||||
response.status()
|
||||
));
|
||||
}
|
||||
|
||||
let text = response.text().await?;
|
||||
let filename = tarball_url.rsplit('/').next().unwrap_or("");
|
||||
expected_digest_from_checksums(&text, filename).ok_or_else(|| {
|
||||
anyhow!(
|
||||
"checksum artifact {} does not list {}",
|
||||
checksum_url,
|
||||
filename
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Extract the digest for `filename` from a sha256sum-style checksum file
|
||||
/// (lines of `<hex> <name>` or `<hex> *<name>`).
|
||||
fn expected_digest_from_checksums(text: &str, filename: &str) -> Option<String> {
|
||||
for line in text.lines() {
|
||||
let mut fields = line.split_whitespace();
|
||||
let (Some(digest), Some(name)) = (fields.next(), fields.next()) else {
|
||||
continue;
|
||||
};
|
||||
if name.trim_start_matches('*') == filename {
|
||||
return Some(digest.to_lowercase());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
async fn download_file_async(
|
||||
url: &str,
|
||||
dest: &Path,
|
||||
progress: &dyn Progress,
|
||||
expected: Option<&str>,
|
||||
) -> Result<()> {
|
||||
log::debug!("Downloading: {}", url);
|
||||
|
||||
let client = Client::builder()
|
||||
@@ -148,8 +288,10 @@ async fn download_file_async(url: &str, dest: &Path, progress: &dyn Progress) ->
|
||||
let temp_dest = dest.with_extension("partial");
|
||||
|
||||
// Download into the temp file, then atomically rename to the final path.
|
||||
// On any failure after the temp file is created we remove it so stale
|
||||
// .partial files don't accumulate in the cache directory.
|
||||
// Verification runs before the rename so a corrupted download never
|
||||
// reaches the cache; on any failure after the temp file is created we
|
||||
// remove it so stale .partial files don't accumulate.
|
||||
let mut hasher = DownloadHasher::new();
|
||||
let result: Result<()> = async {
|
||||
let mut file = tokio::fs::File::create(&temp_dest)
|
||||
.await
|
||||
@@ -159,10 +301,15 @@ async fn download_file_async(url: &str, dest: &Path, progress: &dyn Progress) ->
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.context("Download interrupted")?;
|
||||
tokio::io::copy(&mut &chunk[..], &mut file).await?;
|
||||
hasher.update(&chunk);
|
||||
progress.advance(chunk.len() as u64);
|
||||
}
|
||||
|
||||
progress.finish();
|
||||
hasher.verify(expected)?;
|
||||
if expected.is_some() {
|
||||
log::debug!("Verified content digest: {}", dest.display());
|
||||
}
|
||||
|
||||
tokio::fs::rename(&temp_dest, dest)
|
||||
.await
|
||||
@@ -321,6 +468,15 @@ async fn try_download_oci_image(
|
||||
}
|
||||
|
||||
let arch_body = arch_response.text().await?;
|
||||
|
||||
// The arch manifest was requested by digest: verify the response
|
||||
// matches, since the layer digests read from it are trusted next.
|
||||
let mut manifest_hasher = DownloadHasher::new();
|
||||
manifest_hasher.update(arch_body.as_bytes());
|
||||
manifest_hasher
|
||||
.verify(Some(manifest_digest))
|
||||
.context("architecture manifest failed digest verification")?;
|
||||
|
||||
let arch_manifest: Value = serde_json::from_str(&arch_body)
|
||||
.context("Failed to parse architecture manifest JSON")?;
|
||||
|
||||
@@ -378,7 +534,16 @@ async fn try_download_oci_image(
|
||||
|
||||
let layer_path = temp_dir.path().join(&layer_name);
|
||||
let label = format!("Layer {}/{} ({})", i + 1, layers.len(), layer_name);
|
||||
download_blob_with_auth(client, &blob_url, &layer_path, &token, &label, progress).await?;
|
||||
download_blob_with_auth(
|
||||
client,
|
||||
&blob_url,
|
||||
&layer_path,
|
||||
&token,
|
||||
&label,
|
||||
layer_digest,
|
||||
progress,
|
||||
)
|
||||
.await?;
|
||||
layer_names.push(layer_name);
|
||||
}
|
||||
|
||||
@@ -561,6 +726,7 @@ async fn download_blob_with_auth(
|
||||
dest: &Path,
|
||||
token: &str,
|
||||
label: &str,
|
||||
expected_digest: &str,
|
||||
progress: &dyn Progress,
|
||||
) -> Result<()> {
|
||||
let mut request = client.get(url);
|
||||
@@ -585,6 +751,7 @@ async fn download_blob_with_auth(
|
||||
.await
|
||||
.context("Failed to create destination file")?;
|
||||
|
||||
let mut hasher = DownloadHasher::new();
|
||||
let mut stream = response.bytes_stream();
|
||||
use futures_util::StreamExt;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
@@ -594,12 +761,23 @@ async fn download_blob_with_auth(
|
||||
file.write_all(&chunk)
|
||||
.await
|
||||
.context("Failed to write chunk")?;
|
||||
hasher.update(&chunk);
|
||||
progress.advance(chunk.len() as u64);
|
||||
}
|
||||
|
||||
file.flush().await?;
|
||||
progress.finish();
|
||||
|
||||
// The blob was addressed by this digest in the manifest; a mismatch means
|
||||
// truncated or tampered content. The blob lives in a temp directory that
|
||||
// is discarded on error, so a bad layer never reaches the cache.
|
||||
hasher.verify(Some(expected_digest)).with_context(|| {
|
||||
format!(
|
||||
"layer blob {} failed verification against manifest digest",
|
||||
url
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -626,7 +804,81 @@ fn media_type_to_extension(media_type: &str) -> &'static str {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::expected_digest_from_checksums;
|
||||
use super::parse_www_authenticate;
|
||||
use super::DownloadHasher;
|
||||
|
||||
// sha256("abc") and sha512("abc") — standard test vectors
|
||||
const SHA256_ABC: &str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad";
|
||||
const SHA512_ABC: &str = "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a\
|
||||
2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f";
|
||||
|
||||
fn hasher_with(content: &[u8]) -> DownloadHasher {
|
||||
let mut hasher = DownloadHasher::new();
|
||||
hasher.update(content);
|
||||
hasher
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_oci_digests() {
|
||||
let h = hasher_with(b"abc");
|
||||
assert!(h.verify(Some(&format!("sha256:{}", SHA256_ABC))).is_ok());
|
||||
assert!(h.verify(Some(&format!("sha512:{}", SHA512_ABC))).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_bare_hex_is_sha256() {
|
||||
// distro checksum files carry a bare sha256 hex string
|
||||
let h = hasher_with(b"abc");
|
||||
assert!(h.verify(Some(SHA256_ABC)).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_mismatch_errors() {
|
||||
let h = hasher_with(b"abc");
|
||||
let wrong = format!("sha256:{}", "0".repeat(64));
|
||||
assert!(h.verify(Some(&wrong)).is_err());
|
||||
// right hash, wrong algorithm name pinned to it
|
||||
let algo_mismatch = format!("sha512:{}", SHA256_ABC);
|
||||
assert!(h.verify(Some(&algo_mismatch)).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_none_and_unknown_algo_accept() {
|
||||
let h = hasher_with(b"abc");
|
||||
assert!(h.verify(None).is_ok());
|
||||
assert!(h.verify(Some("md5:0123456789abcdef")).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checksum_parsing_bsd_style() {
|
||||
// Alpine's per-file .sha256: "<hex> <filename>"
|
||||
let text = format!("{} alpine-minirootfs-3.23.0-x86_64.tar.gz\n", SHA256_ABC);
|
||||
assert_eq!(
|
||||
expected_digest_from_checksums(&text, "alpine-minirootfs-3.23.0-x86_64.tar.gz"),
|
||||
Some(SHA256_ABC.to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checksum_parsing_binary_marker() {
|
||||
// Ubuntu's SHA256SUMS: "<hex> *<filename>" (binary-mode marker)
|
||||
let text = format!("{} *noble-base-amd64.tar.gz\n", SHA256_ABC);
|
||||
assert_eq!(
|
||||
expected_digest_from_checksums(&text, "noble-base-amd64.tar.gz"),
|
||||
Some(SHA256_ABC.to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checksum_parsing_missing_and_malformed() {
|
||||
let text = format!("{} *other-file.tar.gz\nnot-a-checksum-line\n", SHA256_ABC);
|
||||
assert_eq!(
|
||||
expected_digest_from_checksums(&text, "noble-base-amd64.tar.gz"),
|
||||
None
|
||||
);
|
||||
assert_eq!(expected_digest_from_checksums("", "x"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_docker_hub_challenge() {
|
||||
|
||||
Reference in New Issue
Block a user