mirror of
https://github.com/vhaudiquet/ecrou.git
synced 2026-11-12 22:22:44 +00:00
feat: port the image machinery from ecr
ecrou's workspace starts from ecr's confinement-neutral half: image reference resolution, the content cache with digest freshness, layer extraction with whiteouts and deferred directory modes, provisioned hot cells, and the progress/error/config plumbing. Adaptations: the cache lives in ~/.cache/ecrou, the namespace-era error variants and utils constants are gone, KvmUnavailable arrives ahead of its caller, rootfs loses its exec() delegation, and uname(2) goes through libc now that nix is no longer a dependency. The CLI crate is a stub until its commit.
This commit is contained in:
1 parent
2bda5f8554
commit
9855776847
15 files changed
+5585
No files matched your search
@@ -0,0 +1,27 @@
|
||||
[package]
|
||||
name = "ecrou-cli"
|
||||
description = "Enter one-shot microVM environments"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
authors.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "ecrou"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
ecrou = { path = "../ecrou" }
|
||||
|
||||
# CLI parsing
|
||||
clap = { version = "4", features = ["derive", "env"] }
|
||||
|
||||
# Error handling
|
||||
anyhow = "1"
|
||||
|
||||
# Logging facade; the stderr sink lives in src/logger.rs
|
||||
log = { version = "0.4", features = ["std"] }
|
||||
|
||||
# Progress bar rendering of library progress events
|
||||
indicatif = "0.18"
|
||||
@@ -0,0 +1 @@
|
||||
fn main() {}
|
||||
@@ -0,0 +1,45 @@
|
||||
[package]
|
||||
name = "ecrou"
|
||||
description = "One-shot microVM environments on libkrun"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
authors.workspace = true
|
||||
|
||||
[dependencies]
|
||||
# Config parsing
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_yaml = "0.9"
|
||||
|
||||
# HTTP downloads
|
||||
reqwest = { version = "0.13", features = ["blocking", "stream"] }
|
||||
|
||||
# Tarball extraction
|
||||
tar = "0.4"
|
||||
flate2 = "1"
|
||||
xz2 = "0.1"
|
||||
zstd = "0.13"
|
||||
|
||||
# Download integrity: verify content against registry/CDN digests
|
||||
sha2 = "0.10"
|
||||
|
||||
# Logging facade (the CLI installs the sink)
|
||||
log = "0.4"
|
||||
|
||||
# Temp directories (the per-run scratch rootfs)
|
||||
tempfile = "3"
|
||||
|
||||
# Error handling
|
||||
anyhow = "1"
|
||||
thiserror = "2"
|
||||
|
||||
# Utilities
|
||||
dirs = "6"
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "io-util"] }
|
||||
futures-util = "0.3"
|
||||
serde_json = "1"
|
||||
|
||||
# libkrun FFI: raw ioctl for the KVM probe, runtime dlopen for the ABI
|
||||
libc = "0.2"
|
||||
libloading = "0.8"
|
||||
@@ -0,0 +1,43 @@
|
||||
use anyhow::Context;
|
||||
|
||||
use crate::Result;
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
pub struct Config {
|
||||
#[serde(default)]
|
||||
pub dns: Vec<String>,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn load() -> Result<Self> {
|
||||
let config_path = dirs::config_dir().map(|p| p.join("ecrou.yaml"));
|
||||
|
||||
match config_path {
|
||||
Some(path) if path.exists() => {
|
||||
let content =
|
||||
std::fs::read_to_string(&path).context("Failed to read config file")?;
|
||||
|
||||
let config: Config =
|
||||
serde_yaml::from_str(&content).context("Failed to parse config file")?;
|
||||
|
||||
// Set defaults
|
||||
let config = Config {
|
||||
dns: if config.dns.is_empty() {
|
||||
vec!["1.1.1.1".to_string()]
|
||||
} else {
|
||||
config.dns
|
||||
},
|
||||
};
|
||||
|
||||
Ok(config)
|
||||
}
|
||||
_ => {
|
||||
// No config file, use defaults
|
||||
Ok(Config {
|
||||
dns: vec!["1.1.1.1".to_string()],
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,534 @@
|
||||
use anyhow::{anyhow, Context};
|
||||
|
||||
use crate::Result;
|
||||
|
||||
/// Known distributions with optimized direct tarball downloads
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub enum Distro {
|
||||
Ubuntu,
|
||||
Alpine,
|
||||
}
|
||||
|
||||
/// Represents the source of the container image
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum ImageSource {
|
||||
/// Direct tarball download from known distro
|
||||
DirectTarball {
|
||||
distro: Distro,
|
||||
version: Option<String>,
|
||||
},
|
||||
/// OCI/Docker registry image
|
||||
OciImage {
|
||||
registry: String,
|
||||
repository: String,
|
||||
tag: String,
|
||||
architecture: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl Distro {
|
||||
pub fn from_name(name: &str) -> Result<Self> {
|
||||
match name.to_lowercase().as_str() {
|
||||
"ubuntu" => Ok(Distro::Ubuntu),
|
||||
"alpine" => Ok(Distro::Alpine),
|
||||
_ => Err(anyhow!("Unknown distribution: {}", name).into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse image reference and return the appropriate source
|
||||
/// Supports:
|
||||
/// - Simple distro names: ubuntu, debian, alpine
|
||||
/// - Distro with version: ubuntu:noble, alpine:3.19
|
||||
/// - OCI image references: docker://ubuntu:latest, quay.io/centos/centos:stream9
|
||||
/// - Docker Hub shorthand: ubuntu:latest (without registry prefix)
|
||||
pub fn parse_image_ref(input: &str, arch: &str) -> Result<ImageSource> {
|
||||
let input = input.trim();
|
||||
|
||||
// Check for explicit docker:// or oci:// prefix
|
||||
if let Some(rest) = input
|
||||
.strip_prefix("docker://")
|
||||
.or_else(|| input.strip_prefix("oci://"))
|
||||
{
|
||||
return Ok(parse_oci_ref(rest, arch)?);
|
||||
}
|
||||
|
||||
// Check for registry prefix (contains /)
|
||||
if input.contains('/') {
|
||||
return Ok(parse_oci_ref(input, arch)?);
|
||||
}
|
||||
|
||||
// Try to parse as known distro
|
||||
let (name, version) = match input.split_once(':') {
|
||||
Some((n, v)) => (n, Some(v.to_string())),
|
||||
None => (input, None),
|
||||
};
|
||||
|
||||
// Check if it's a known distro with optimized path
|
||||
match name.to_lowercase().as_str() {
|
||||
"ubuntu" | "alpine" => {
|
||||
let distro = Distro::from_name(name)?;
|
||||
Ok(ImageSource::DirectTarball { distro, version })
|
||||
}
|
||||
// Arch: use Docker image (has mirrors configured, unlike bootstrap tarball)
|
||||
"arch" => {
|
||||
let oci_arch = map_oci_arch(arch);
|
||||
Ok(ImageSource::OciImage {
|
||||
registry: "docker.io".to_string(),
|
||||
repository: "library/archlinux".to_string(),
|
||||
tag: version.unwrap_or_else(|| "latest".to_string()),
|
||||
architecture: oci_arch,
|
||||
})
|
||||
}
|
||||
// Special case: gentoo maps to gentoo/stage3 on Docker Hub (full rootfs)
|
||||
"gentoo" => {
|
||||
let oci_arch = map_oci_arch(arch);
|
||||
Ok(ImageSource::OciImage {
|
||||
registry: "docker.io".to_string(),
|
||||
repository: "gentoo/stage3".to_string(),
|
||||
tag: version.unwrap_or_else(|| "latest".to_string()),
|
||||
architecture: oci_arch,
|
||||
})
|
||||
}
|
||||
// Default to Docker Hub for unknown distros (debian, fedora, etc.)
|
||||
_ => Ok(parse_oci_ref(input, arch)?),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse OCI image reference (registry/repo:tag or repo:tag)
|
||||
fn parse_oci_ref(input: &str, arch: &str) -> anyhow::Result<ImageSource> {
|
||||
// Split off the tag. A ':' is a tag separator only when it appears after
|
||||
// the last '/'; any ':' before a '/' is a port number
|
||||
// (e.g. localhost:5000/image). With no '/' the single ':' is the tag.
|
||||
let (without_tag, tag) = if let Some(slash_pos) = input.rfind('/') {
|
||||
let after_slash = &input[slash_pos + 1..];
|
||||
if let Some(colon_pos) = after_slash.find(':') {
|
||||
(
|
||||
&input[..slash_pos + 1 + colon_pos],
|
||||
input[slash_pos + 1 + colon_pos + 1..].to_string(),
|
||||
)
|
||||
} else {
|
||||
(input, "latest".to_string())
|
||||
}
|
||||
} else {
|
||||
// No slash: bare "ubuntu" or "ubuntu:latest"
|
||||
match input.split_once(':') {
|
||||
Some((name, t)) => (name, t.to_string()),
|
||||
None => (input, "latest".to_string()),
|
||||
}
|
||||
};
|
||||
|
||||
// Split without_tag into registry + repository.
|
||||
// The first path component is the registry when it contains '.' or ':'
|
||||
// (hostname / host:port) or is the literal "localhost".
|
||||
let (registry, repository) = if let Some((first, rest)) = without_tag.split_once('/') {
|
||||
if first.contains('.') || first.contains(':') || first == "localhost" {
|
||||
(first.to_string(), rest.to_string())
|
||||
} else {
|
||||
// Org-qualified Docker Hub shorthand: "myorg/myimage"
|
||||
("docker.io".to_string(), without_tag.to_string())
|
||||
}
|
||||
} else {
|
||||
// Bare image name → Docker Hub library image
|
||||
("docker.io".to_string(), format!("library/{}", without_tag))
|
||||
};
|
||||
|
||||
// Map architecture to OCI standard
|
||||
let oci_arch = map_oci_arch(arch);
|
||||
|
||||
Ok(ImageSource::OciImage {
|
||||
registry,
|
||||
repository,
|
||||
tag,
|
||||
architecture: oci_arch,
|
||||
})
|
||||
}
|
||||
|
||||
/// Map architecture to OCI standard names
|
||||
pub fn map_oci_arch(arch: &str) -> String {
|
||||
crate::utils::map_oci_arch(arch)
|
||||
}
|
||||
|
||||
/// Map ecrou architecture names to distro-specific names
|
||||
pub fn map_arch(distro: Distro, arch: &str) -> String {
|
||||
let distro_name = match distro {
|
||||
Distro::Ubuntu => "ubuntu",
|
||||
Distro::Alpine => "alpine",
|
||||
};
|
||||
crate::utils::map_arch_for_distro(distro_name, arch)
|
||||
}
|
||||
|
||||
/// Resolve the download URL for a known distro (optimized path)
|
||||
pub fn resolve_distro_url(distro: &Distro, version: Option<&str>, arch: &str) -> Result<String> {
|
||||
Ok(match distro {
|
||||
Distro::Ubuntu => resolve_ubuntu_url(version, arch),
|
||||
Distro::Alpine => resolve_alpine_url(version, arch),
|
||||
}?)
|
||||
}
|
||||
|
||||
/// URL of the checksum artifact covering the tarball at `tarball_url`, when
|
||||
/// the distro publishes one. Alpine ships a per-file `.sha256`; Ubuntu
|
||||
/// publishes a `SHA256SUMS` manifest per release directory.
|
||||
pub fn checksum_artifact_url(distro: &Distro, tarball_url: &str) -> Option<String> {
|
||||
match distro {
|
||||
Distro::Alpine => Some(format!("{}.sha256", tarball_url)),
|
||||
Distro::Ubuntu => {
|
||||
let dir = tarball_url.rsplit_once('/')?.0;
|
||||
Some(format!("{}/SHA256SUMS", dir))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch the latest Ubuntu codename from a changelogs.ubuntu.com meta-release file.
|
||||
/// Pass the LTS-only URL to get the latest LTS, or the full URL for the latest release.
|
||||
fn fetch_ubuntu_codename(meta_release_url: &str) -> anyhow::Result<String> {
|
||||
let text = reqwest::blocking::get(meta_release_url)
|
||||
.with_context(|| format!("Failed to fetch {}", meta_release_url))?
|
||||
.text()
|
||||
.with_context(|| format!("Failed to read {}", meta_release_url))?;
|
||||
|
||||
let mut current_dist: Option<String> = None;
|
||||
let mut latest: Option<String> = None;
|
||||
|
||||
for line in text.lines() {
|
||||
if let Some(dist) = line.strip_prefix("Dist: ") {
|
||||
current_dist = Some(dist.trim().to_string());
|
||||
} else if line.trim_start().starts_with("Supported: 1") {
|
||||
if let Some(dist) = current_dist.take() {
|
||||
latest = Some(dist);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
latest.ok_or_else(|| {
|
||||
anyhow!(
|
||||
"Could not determine Ubuntu codename from {}",
|
||||
meta_release_url
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Fetch the current Alpine minirootfs version from latest-releases.yaml on the CDN.
|
||||
/// The `latest-stable/` directory is a server-side symlink; the YAML file it contains
|
||||
/// tells us the exact version number needed for the tarball filename.
|
||||
/// Fetch the latest minirootfs version for a given Alpine CDN branch.
|
||||
/// `branch` is the directory name on the CDN, e.g. `"latest-stable"` or `"v3.23"`.
|
||||
fn fetch_alpine_version_from_branch(branch: &str, arch: &str) -> anyhow::Result<String> {
|
||||
#[derive(serde::Deserialize)]
|
||||
struct AlpineRelease {
|
||||
file: Option<String>,
|
||||
version: Option<String>,
|
||||
}
|
||||
|
||||
let url = format!(
|
||||
"https://dl-cdn.alpinelinux.org/alpine/{}/releases/{}/latest-releases.yaml",
|
||||
branch, arch
|
||||
);
|
||||
let text = reqwest::blocking::get(&url)
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"Failed to fetch Alpine latest-releases.yaml for branch {}",
|
||||
branch
|
||||
)
|
||||
})?
|
||||
.text()
|
||||
.context("Failed to read Alpine latest-releases.yaml")?;
|
||||
|
||||
let releases: Vec<AlpineRelease> =
|
||||
serde_yaml::from_str(&text).context("Failed to parse Alpine latest-releases.yaml")?;
|
||||
|
||||
for release in releases {
|
||||
let is_minirootfs = release
|
||||
.file
|
||||
.as_deref()
|
||||
.map(|f| f.contains("minirootfs"))
|
||||
.unwrap_or(false);
|
||||
if is_minirootfs {
|
||||
// Prefer explicit `version:` field; fall back to parsing the filename.
|
||||
// Filename format: alpine-minirootfs-3.23.0-x86_64.tar.gz
|
||||
if let Some(v) = release.version {
|
||||
return Ok(v);
|
||||
}
|
||||
if let Some(v) = release.file.as_deref().and_then(|f| f.split('-').nth(2)) {
|
||||
return Ok(v.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(anyhow!(
|
||||
"Could not find minirootfs entry in Alpine latest-releases.yaml for branch {}",
|
||||
branch
|
||||
))
|
||||
}
|
||||
|
||||
fn fetch_alpine_latest_version(arch: &str) -> anyhow::Result<String> {
|
||||
fetch_alpine_version_from_branch("latest-stable", arch)
|
||||
}
|
||||
|
||||
/// Resolve a `major.minor` Alpine series (e.g. `"3.24"`) to its current
|
||||
/// patch release by querying the CDN branch `v{minor}`.
|
||||
fn fetch_alpine_minor_version(minor: &str, arch: &str) -> anyhow::Result<String> {
|
||||
fetch_alpine_version_from_branch(&format!("v{}", minor), arch)
|
||||
}
|
||||
|
||||
/// Resolve the canonical version string for a distro, performing a network lookup
|
||||
/// only when the requested version is a floating alias (e.g. "latest", "lts").
|
||||
/// The returned string is suitable for use as a stable cache key.
|
||||
pub fn resolve_distro_version(
|
||||
distro: &Distro,
|
||||
version: Option<&str>,
|
||||
arch: &str,
|
||||
) -> Result<String> {
|
||||
Ok(match distro {
|
||||
Distro::Ubuntu => resolve_ubuntu_version(version),
|
||||
Distro::Alpine => resolve_alpine_version(version, arch),
|
||||
}?)
|
||||
}
|
||||
|
||||
/// Build a map of YY.MM version strings to codenames by parsing the Ubuntu
|
||||
/// meta-release file (e.g. "24.04" → "noble", "22.04" → "jammy").
|
||||
/// Uses the full meta-release (not -lts) so non-LTS versions are also covered.
|
||||
fn fetch_ubuntu_version_map() -> anyhow::Result<std::collections::HashMap<String, String>> {
|
||||
let text = reqwest::blocking::get("https://changelogs.ubuntu.com/meta-release")
|
||||
.context("Failed to fetch Ubuntu meta-release")?
|
||||
.text()
|
||||
.context("Failed to read Ubuntu meta-release")?;
|
||||
|
||||
let mut map = std::collections::HashMap::new();
|
||||
let mut current_dist: Option<String> = None;
|
||||
|
||||
for line in text.lines() {
|
||||
if let Some(dist) = line.strip_prefix("Dist: ") {
|
||||
current_dist = Some(dist.trim().to_string());
|
||||
} else if let Some(version_str) = line.strip_prefix("Version: ") {
|
||||
// Version field may be "22.04", "22.04 LTS", or "24.04.1 LTS".
|
||||
// Normalise to YY.MM by taking the first two dot-separated components.
|
||||
let raw = version_str.split_whitespace().next().unwrap_or("");
|
||||
let normalised: String = {
|
||||
let mut parts = raw.splitn(3, '.');
|
||||
match (parts.next(), parts.next()) {
|
||||
(Some(a), Some(b)) => format!("{}.{}", a, b),
|
||||
_ => raw.to_string(),
|
||||
}
|
||||
};
|
||||
if let Some(dist) = ¤t_dist {
|
||||
map.insert(normalised, dist.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(map)
|
||||
}
|
||||
|
||||
fn resolve_ubuntu_version(version: Option<&str>) -> anyhow::Result<String> {
|
||||
let version = version.unwrap_or("latest");
|
||||
match version {
|
||||
"latest" => fetch_ubuntu_codename("https://changelogs.ubuntu.com/meta-release"),
|
||||
"lts" | "latest-lts" => {
|
||||
fetch_ubuntu_codename("https://changelogs.ubuntu.com/meta-release-lts")
|
||||
}
|
||||
other => {
|
||||
// If it looks like a YY.MM version number, resolve it to a codename
|
||||
// via the meta-release file so the mapping never goes stale.
|
||||
if is_ubuntu_version_number(other) {
|
||||
let map = fetch_ubuntu_version_map()?;
|
||||
map.get(other).cloned().ok_or_else(|| {
|
||||
anyhow!(
|
||||
"Unknown Ubuntu version '{}'. \
|
||||
Use the codename directly (e.g. noble, jammy) or check \
|
||||
https://changelogs.ubuntu.com/meta-release",
|
||||
other
|
||||
)
|
||||
})
|
||||
} else {
|
||||
// Treat as a codename and pass through (e.g. "noble", "jammy")
|
||||
Ok(other.to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true for strings of the form "YY.MM" (two numeric dot-separated components).
|
||||
fn is_ubuntu_version_number(s: &str) -> bool {
|
||||
let mut parts = s.splitn(3, '.');
|
||||
matches!(
|
||||
(parts.next(), parts.next(), parts.next()),
|
||||
(Some(a), Some(b), None)
|
||||
if !a.is_empty() && !b.is_empty()
|
||||
&& a.chars().all(|c| c.is_ascii_digit())
|
||||
&& b.chars().all(|c| c.is_ascii_digit())
|
||||
)
|
||||
}
|
||||
|
||||
fn resolve_alpine_version(version: Option<&str>, arch: &str) -> anyhow::Result<String> {
|
||||
let alpine_arch = map_arch(Distro::Alpine, arch);
|
||||
Ok(match version.unwrap_or("latest") {
|
||||
"latest" | "stable" => fetch_alpine_latest_version(&alpine_arch)?,
|
||||
// edge is a rolling branch; query the CDN to get the current
|
||||
// date-stamped version (e.g. "20250401") so the URL and cache key
|
||||
// are correct. resolve_alpine_url maps this date string back to the
|
||||
// "edge" CDN directory via the all-digits guard in its release match.
|
||||
"edge" => fetch_alpine_version_from_branch("edge", &alpine_arch)?,
|
||||
v => {
|
||||
// "3.23" — one dot: major.minor series → fetch current patch from CDN
|
||||
// "3.23.0" — two dots: full version already → pass through as-is
|
||||
let dots = v.chars().filter(|&c| c == '.').count();
|
||||
if dots == 1 {
|
||||
fetch_alpine_minor_version(v, &alpine_arch)?
|
||||
} else {
|
||||
v.to_string()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn resolve_ubuntu_url(version: Option<&str>, arch: &str) -> anyhow::Result<String> {
|
||||
let codename = resolve_ubuntu_version(version)?;
|
||||
let arch = map_arch(Distro::Ubuntu, arch);
|
||||
Ok(format!(
|
||||
"https://cdimage.ubuntu.com/ubuntu-base/{}/daily/current/{}-base-{}.tar.gz",
|
||||
codename, codename, arch
|
||||
))
|
||||
}
|
||||
|
||||
fn resolve_alpine_url(version: Option<&str>, arch: &str) -> anyhow::Result<String> {
|
||||
let version_str = version.unwrap_or("latest");
|
||||
let alpine_arch = map_arch(Distro::Alpine, arch);
|
||||
let version_num = resolve_alpine_version(Some(version_str), arch)?;
|
||||
|
||||
// Derive the CDN release directory from the version string.
|
||||
// After resolve_distro_version runs, version_str may already be the
|
||||
// *resolved* value rather than the original alias:
|
||||
// "latest"/"stable" → e.g. "3.23.1" (dots present → v3.23 below)
|
||||
// "edge" → e.g. "20250401" (all digits, no dots)
|
||||
// "3.23" → e.g. "3.23.1" (dots present → v3.23 below)
|
||||
// The all-digit check catches resolved edge dates and maps them back to
|
||||
// the "edge" CDN directory.
|
||||
let release = match version_str {
|
||||
"latest" | "stable" => "latest-stable".to_string(),
|
||||
"edge" => "edge".to_string(),
|
||||
v if v.chars().all(|c| c.is_ascii_digit()) => "edge".to_string(),
|
||||
other => {
|
||||
// "3.23" or "3.23.1" → "v3.23"
|
||||
let mut parts = other.splitn(3, '.');
|
||||
let major = parts.next().unwrap_or("0");
|
||||
let minor = parts.next().unwrap_or("0");
|
||||
format!("v{}.{}", major, minor)
|
||||
}
|
||||
};
|
||||
|
||||
Ok(format!(
|
||||
"https://dl-cdn.alpinelinux.org/alpine/{}/releases/{}/alpine-minirootfs-{}-{}.tar.gz",
|
||||
release, alpine_arch, version_num, alpine_arch
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn checksum_artifact_url_per_distro() {
|
||||
assert_eq!(
|
||||
checksum_artifact_url(
|
||||
&Distro::Alpine,
|
||||
"https://dl-cdn.alpinelinux.org/alpine/v3.23/releases/x86_64/alpine-minirootfs-3.23.0-x86_64.tar.gz"
|
||||
)
|
||||
.as_deref(),
|
||||
Some(
|
||||
"https://dl-cdn.alpinelinux.org/alpine/v3.23/releases/x86_64/alpine-minirootfs-3.23.0-x86_64.tar.gz.sha256"
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
checksum_artifact_url(
|
||||
&Distro::Ubuntu,
|
||||
"https://cdimage.ubuntu.com/ubuntu-base/noble/daily/current/noble-base-amd64.tar.gz"
|
||||
)
|
||||
.as_deref(),
|
||||
Some("https://cdimage.ubuntu.com/ubuntu-base/noble/daily/current/SHA256SUMS")
|
||||
);
|
||||
}
|
||||
|
||||
fn oci(registry: &str, repository: &str, tag: &str) -> ImageSource {
|
||||
ImageSource::OciImage {
|
||||
registry: registry.to_string(),
|
||||
repository: repository.to_string(),
|
||||
tag: tag.to_string(),
|
||||
architecture: "amd64".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse(input: &str) -> ImageSource {
|
||||
parse_oci_ref(input, "amd64").expect("parse failed")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bare_name() {
|
||||
// "ubuntu" → Docker Hub library, tag=latest
|
||||
let src = parse("ubuntu");
|
||||
assert_eq!(src, oci("docker.io", "library/ubuntu", "latest"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_name_with_tag() {
|
||||
let src = parse("ubuntu:noble");
|
||||
assert_eq!(src, oci("docker.io", "library/ubuntu", "noble"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_org_repo() {
|
||||
let src = parse("myorg/myimage:v2");
|
||||
assert_eq!(src, oci("docker.io", "myorg/myimage", "v2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_registry_with_port_no_tag() {
|
||||
// The colon in "localhost:5000" must NOT be treated as a tag separator
|
||||
let src = parse("localhost:5000/myimage");
|
||||
assert_eq!(src, oci("localhost:5000", "myimage", "latest"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_registry_with_port_and_tag() {
|
||||
let src = parse("localhost:5000/myimage:v1");
|
||||
assert_eq!(src, oci("localhost:5000", "myimage", "v1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_registry_with_port_and_org() {
|
||||
let src = parse("localhost:5000/org/myimage:v1");
|
||||
assert_eq!(src, oci("localhost:5000", "org/myimage", "v1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_named_registry() {
|
||||
let src = parse("quay.io/centos/centos:stream9");
|
||||
assert_eq!(src, oci("quay.io", "centos/centos", "stream9"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_docker_hub_fqdn() {
|
||||
let src = parse("registry-1.docker.io/library/ubuntu:noble");
|
||||
assert_eq!(src, oci("registry-1.docker.io", "library/ubuntu", "noble"));
|
||||
}
|
||||
|
||||
/// After resolve_distro_version, alpine:edge carries a date string like
|
||||
/// "20250401". resolve_alpine_url must still produce a URL under the
|
||||
/// "edge" CDN directory, not a bogus "v20250401.0" directory.
|
||||
#[test]
|
||||
fn test_alpine_edge_resolved_date_uses_edge_directory() {
|
||||
// Simulate the already-resolved version that main.rs stores after calling
|
||||
// resolve_distro_version("edge", …).
|
||||
let url = resolve_alpine_url(Some("20250401"), "amd64").unwrap();
|
||||
assert!(
|
||||
url.contains("/alpine/edge/"),
|
||||
"expected URL to contain '/alpine/edge/' but got: {}",
|
||||
url
|
||||
);
|
||||
assert!(
|
||||
url.contains("minirootfs-20250401-"),
|
||||
"expected URL to contain 'minirootfs-20250401-' but got: {}",
|
||||
url
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,967 @@
|
||||
use crate::progress::{Progress, Unit};
|
||||
use anyhow::{anyhow, Context};
|
||||
use futures_util::StreamExt;
|
||||
use reqwest::Client;
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256, Sha512};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::distro::{Distro, ImageSource};
|
||||
use crate::Result;
|
||||
|
||||
/// Path of the digest sidecar file for a cached OCI image.
|
||||
/// e.g. `foo.tar.gz` → `foo.tar.gz.digest`
|
||||
pub fn digest_sidecar(cache_path: &Path) -> PathBuf {
|
||||
let mut s = cache_path.as_os_str().to_owned();
|
||||
s.push(".digest");
|
||||
PathBuf::from(s)
|
||||
}
|
||||
|
||||
/// Incremental sha256+sha512 hasher used to verify downloaded content against
|
||||
/// content-addressed digests. Both algorithms run side by side so the caller
|
||||
/// can check whatever digest the manifest or checksum file names.
|
||||
struct DownloadHasher {
|
||||
sha256: Sha256,
|
||||
sha512: Sha512,
|
||||
}
|
||||
|
||||
impl DownloadHasher {
|
||||
fn new() -> Self {
|
||||
DownloadHasher {
|
||||
sha256: Sha256::new(),
|
||||
sha512: Sha512::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn update(&mut self, chunk: &[u8]) {
|
||||
self.sha256.update(chunk);
|
||||
self.sha512.update(chunk);
|
||||
}
|
||||
|
||||
/// Verify the streamed content. `expected` is an OCI `algo:hex` digest or
|
||||
/// a bare sha256 hex string (distro checksum files); None verifies nothing.
|
||||
/// Algorithms outside sha256/sha512 cannot be checked: warn and accept
|
||||
/// rather than reject images a registry may legitimately serve.
|
||||
fn verify(&self, expected: Option<&str>) -> anyhow::Result<()> {
|
||||
let Some(expected) = expected else {
|
||||
return Ok(());
|
||||
};
|
||||
let (algo, hex) = match expected.split_once(':') {
|
||||
Some((algo, hex)) => (algo, hex),
|
||||
None => ("sha256", expected),
|
||||
};
|
||||
|
||||
let actual = match algo {
|
||||
"sha256" => hex_encode(&self.sha256.clone().finalize()),
|
||||
"sha512" => hex_encode(&self.sha512.clone().finalize()),
|
||||
other => {
|
||||
log::warn!(
|
||||
"digest algorithm '{}' not supported; skipping verification",
|
||||
other
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
if actual.eq_ignore_ascii_case(hex) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(anyhow!(
|
||||
"content verification failed: expected {} {}, got {}",
|
||||
algo,
|
||||
hex,
|
||||
actual
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn hex_encode(bytes: &[u8]) -> String {
|
||||
bytes.iter().map(|b| format!("{:02x}", b)).collect()
|
||||
}
|
||||
|
||||
/// Fetch the current manifest digest for an OCI tag without downloading any
|
||||
/// layers. The registry returns the content-addressable digest in the
|
||||
/// `Docker-Content-Digest` response header of any manifest GET.
|
||||
pub fn fetch_oci_digest(registry: &str, repository: &str, tag: &str) -> Result<String> {
|
||||
let digest = tokio::runtime::Runtime::new()
|
||||
.context("Failed to create Tokio runtime")?
|
||||
.block_on(fetch_oci_digest_async(registry, repository, tag))?;
|
||||
Ok(digest)
|
||||
}
|
||||
|
||||
async fn fetch_oci_digest_async(
|
||||
registry: &str,
|
||||
repository: &str,
|
||||
tag: &str,
|
||||
) -> anyhow::Result<String> {
|
||||
let client = Client::builder()
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.context("Failed to create HTTP client")?;
|
||||
|
||||
let token = get_auth_token(&client, registry, repository).await?;
|
||||
|
||||
let manifest_url = if registry == "docker.io" {
|
||||
format!(
|
||||
"https://registry-1.docker.io/v2/{}/manifests/{}",
|
||||
repository, tag
|
||||
)
|
||||
} else {
|
||||
format!("https://{}/v2/{}/manifests/{}", registry, repository, tag)
|
||||
};
|
||||
|
||||
let response = client
|
||||
.get(&manifest_url)
|
||||
.header(
|
||||
"Accept",
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json, \
|
||||
application/vnd.docker.distribution.manifest.v2+json, \
|
||||
application/vnd.oci.image.index.v1+json, \
|
||||
application/vnd.oci.image.manifest.v1+json",
|
||||
)
|
||||
.header("Authorization", format!("Bearer {}", token))
|
||||
.send()
|
||||
.await
|
||||
.context("Failed to fetch manifest for digest check")?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(anyhow!(
|
||||
"Manifest fetch returned HTTP {}",
|
||||
response.status()
|
||||
));
|
||||
}
|
||||
|
||||
response
|
||||
.headers()
|
||||
.get("Docker-Content-Digest")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| anyhow!("Registry did not return a Docker-Content-Digest header"))
|
||||
}
|
||||
|
||||
/// Download a container image (either direct tarball or OCI image),
|
||||
/// reporting transfer progress through `progress`
|
||||
pub fn download_image(
|
||||
source: &ImageSource,
|
||||
dest: &Path,
|
||||
arch: &str,
|
||||
progress: &dyn Progress,
|
||||
) -> Result<()> {
|
||||
// Create a single Tokio runtime for all async I/O in this download.
|
||||
// Previously each of the two sync wrappers (download_file_sync and
|
||||
// download_oci_image) created their own runtime; this consolidates them.
|
||||
let rt = tokio::runtime::Runtime::new().context("Failed to create Tokio runtime")?;
|
||||
|
||||
match source {
|
||||
ImageSource::DirectTarball { distro, version } => {
|
||||
let url = crate::distro::resolve_distro_url(distro, version.as_deref(), arch)?;
|
||||
log::debug!("Resolved URL: {}", url);
|
||||
if !url.starts_with("http://") && !url.starts_with("https://") {
|
||||
return Err(anyhow!("Unsupported URL scheme: {}", url).into());
|
||||
}
|
||||
// Verify the tarball against the checksum the distro publishes
|
||||
// next to it. A fetch failure (missing artifact, network error)
|
||||
// degrades to an unverified download with a warning rather than
|
||||
// hard-failing, mirroring the digest freshness check in rootfs.
|
||||
let expected = match rt.block_on(fetch_published_checksum(distro, &url)) {
|
||||
Ok(expected) => Some(expected),
|
||||
Err(e) => {
|
||||
log::warn!(
|
||||
"could not fetch published checksum ({}); \
|
||||
proceeding without verification",
|
||||
e
|
||||
);
|
||||
None
|
||||
}
|
||||
};
|
||||
Ok(rt.block_on(download_file_async(
|
||||
&url,
|
||||
dest,
|
||||
progress,
|
||||
expected.as_deref(),
|
||||
))?)
|
||||
}
|
||||
ImageSource::OciImage {
|
||||
registry,
|
||||
repository,
|
||||
tag,
|
||||
architecture,
|
||||
} => {
|
||||
log::debug!("Pulling OCI image: {}/{}:{}", registry, repository, tag);
|
||||
Ok(rt.block_on(download_oci_image_async(
|
||||
registry,
|
||||
repository,
|
||||
tag,
|
||||
architecture,
|
||||
dest,
|
||||
progress,
|
||||
))?)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch the sha256 the distro publishes for the tarball at `tarball_url`.
|
||||
async fn fetch_published_checksum(distro: &Distro, tarball_url: &str) -> anyhow::Result<String> {
|
||||
let Some(checksum_url) = crate::distro::checksum_artifact_url(distro, tarball_url) else {
|
||||
return Err(anyhow!("no checksum artifact for {}", tarball_url));
|
||||
};
|
||||
|
||||
let client = Client::builder()
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.context("Failed to create HTTP client")?;
|
||||
|
||||
let response = client
|
||||
.get(&checksum_url)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("Failed to fetch checksum artifact {}", checksum_url))?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(anyhow!(
|
||||
"checksum artifact {} returned HTTP {}",
|
||||
checksum_url,
|
||||
response.status()
|
||||
));
|
||||
}
|
||||
|
||||
let text = response.text().await?;
|
||||
let filename = tarball_url.rsplit('/').next().unwrap_or("");
|
||||
expected_digest_from_checksums(&text, filename).ok_or_else(|| {
|
||||
anyhow!(
|
||||
"checksum artifact {} does not list {}",
|
||||
checksum_url,
|
||||
filename
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Extract the digest for `filename` from a sha256sum-style checksum file
|
||||
/// (lines of `<hex> <name>` or `<hex> *<name>`).
|
||||
fn expected_digest_from_checksums(text: &str, filename: &str) -> Option<String> {
|
||||
for line in text.lines() {
|
||||
let mut fields = line.split_whitespace();
|
||||
let (Some(digest), Some(name)) = (fields.next(), fields.next()) else {
|
||||
continue;
|
||||
};
|
||||
if name.trim_start_matches('*') == filename {
|
||||
return Some(digest.to_lowercase());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
async fn download_file_async(
|
||||
url: &str,
|
||||
dest: &Path,
|
||||
progress: &dyn Progress,
|
||||
expected: Option<&str>,
|
||||
) -> anyhow::Result<()> {
|
||||
log::debug!("Downloading: {}", url);
|
||||
|
||||
let client = Client::builder()
|
||||
.timeout(Duration::from_secs(300))
|
||||
.build()
|
||||
.context("Failed to create HTTP client")?;
|
||||
|
||||
let response = client
|
||||
.get(url)
|
||||
.send()
|
||||
.await
|
||||
.context("Failed to start download")?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(anyhow!("Download failed: HTTP {}", response.status()));
|
||||
}
|
||||
|
||||
let total_size = response.content_length().unwrap_or(0);
|
||||
|
||||
// Create parent directories
|
||||
if let Some(parent) = dest.parent() {
|
||||
tokio::fs::create_dir_all(parent)
|
||||
.await
|
||||
.context("Failed to create cache directory")?;
|
||||
}
|
||||
|
||||
progress.begin(
|
||||
"Downloading",
|
||||
Unit::Bytes,
|
||||
(total_size > 0).then_some(total_size),
|
||||
);
|
||||
|
||||
let temp_dest = dest.with_extension("partial");
|
||||
|
||||
// Download into the temp file, then atomically rename to the final path.
|
||||
// Verification runs before the rename so a corrupted download never
|
||||
// reaches the cache; on any failure after the temp file is created we
|
||||
// remove it so stale .partial files don't accumulate.
|
||||
let mut hasher = DownloadHasher::new();
|
||||
let result: anyhow::Result<()> = async {
|
||||
let mut file = tokio::fs::File::create(&temp_dest)
|
||||
.await
|
||||
.context("Failed to create temporary file")?;
|
||||
|
||||
let mut stream = response.bytes_stream();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.context("Download interrupted")?;
|
||||
tokio::io::copy(&mut &chunk[..], &mut file).await?;
|
||||
hasher.update(&chunk);
|
||||
progress.advance(chunk.len() as u64);
|
||||
}
|
||||
|
||||
progress.finish();
|
||||
hasher.verify(expected)?;
|
||||
if expected.is_some() {
|
||||
log::debug!("Verified content digest: {}", dest.display());
|
||||
}
|
||||
|
||||
tokio::fs::rename(&temp_dest, dest)
|
||||
.await
|
||||
.context("Failed to move partial download to final destination")?;
|
||||
log::debug!("Download complete: {}", dest.display());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
if result.is_err() {
|
||||
// Best-effort cleanup; ignore errors (file may not exist if creation failed).
|
||||
let _ = tokio::fs::remove_file(&temp_dest).await;
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
async fn download_oci_image_async(
|
||||
registry: &str,
|
||||
repository: &str,
|
||||
tag: &str,
|
||||
arch: &str,
|
||||
dest: &Path,
|
||||
progress: &dyn Progress,
|
||||
) -> anyhow::Result<()> {
|
||||
let client = Client::builder()
|
||||
.timeout(Duration::from_secs(300))
|
||||
.build()
|
||||
.context("Failed to create HTTP client")?;
|
||||
|
||||
let digest =
|
||||
try_download_oci_image(&client, registry, repository, tag, arch, dest, progress).await?;
|
||||
|
||||
// Persist the manifest digest so the next run can skip the download when
|
||||
// the tag hasn't changed (see fetch_oci_digest / digest_sidecar).
|
||||
if !digest.is_empty() {
|
||||
let _ = std::fs::write(digest_sidecar(dest), &digest);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn try_download_oci_image(
|
||||
client: &Client,
|
||||
registry: &str,
|
||||
repository: &str,
|
||||
tag: &str,
|
||||
arch: &str,
|
||||
dest: &Path,
|
||||
progress: &dyn Progress,
|
||||
) -> anyhow::Result<String> {
|
||||
// Get authentication token for the registry
|
||||
let token = get_auth_token(client, registry, repository).await?;
|
||||
|
||||
// Construct manifest URL based on registry
|
||||
let manifest_url = if registry == "docker.io" {
|
||||
// Docker Hub uses registry-1.docker.io for API
|
||||
format!(
|
||||
"https://registry-1.docker.io/v2/{}/manifests/{}",
|
||||
repository, tag
|
||||
)
|
||||
} else {
|
||||
format!("https://{}/v2/{}/manifests/{}", registry, repository, tag)
|
||||
};
|
||||
|
||||
log::debug!("Fetching manifest from: {}", manifest_url);
|
||||
|
||||
// Request both manifest list and single manifest types (including OCI formats)
|
||||
let response = client
|
||||
.get(&manifest_url)
|
||||
.header("Accept", "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json")
|
||||
.header("Authorization", format!("Bearer {}", token))
|
||||
.send()
|
||||
.await
|
||||
.context("Failed to get image manifest")?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(anyhow!(
|
||||
"Failed to get manifest: HTTP {}",
|
||||
response.status()
|
||||
));
|
||||
}
|
||||
|
||||
// Capture the tag-level digest before consuming the response body.
|
||||
// This is the content-addressable identity of the manifest (or manifest
|
||||
// list) at this tag — used to detect whether the tag has moved since the
|
||||
// last download.
|
||||
let tag_digest = response
|
||||
.headers()
|
||||
.get("Docker-Content-Digest")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|s| s.to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
let body = response.text().await?;
|
||||
let manifest: Value = serde_json::from_str(&body).context("Failed to parse manifest JSON")?;
|
||||
|
||||
// Check if this is a manifest list (multi-arch)
|
||||
let layers = if let Some(manifests) = manifest["manifests"].as_array() {
|
||||
// This is a manifest list - find the right architecture
|
||||
log::debug!("Got manifest list with {} manifests", manifests.len());
|
||||
|
||||
let manifest_entry = manifests
|
||||
.iter()
|
||||
.find(|m| {
|
||||
let m_arch = m["platform"]["architecture"].as_str().unwrap_or("");
|
||||
let m_os = m["platform"]["os"].as_str().unwrap_or("");
|
||||
m_arch == arch && m_os == "linux"
|
||||
})
|
||||
.ok_or_else(|| {
|
||||
// List available architectures in the error message
|
||||
let available: Vec<&str> = manifests
|
||||
.iter()
|
||||
.filter_map(|m| m["platform"]["architecture"].as_str())
|
||||
.collect();
|
||||
anyhow!(
|
||||
"No manifest found for architecture '{}'. Available: {}",
|
||||
arch,
|
||||
available.join(", ")
|
||||
)
|
||||
})?;
|
||||
|
||||
let manifest_digest = manifest_entry["digest"]
|
||||
.as_str()
|
||||
.ok_or_else(|| anyhow!("No digest in manifest entry"))?;
|
||||
|
||||
log::debug!("Found manifest digest: {}", manifest_digest);
|
||||
|
||||
// Now get the actual manifest for this architecture
|
||||
let arch_manifest_url = if registry == "docker.io" {
|
||||
format!(
|
||||
"https://registry-1.docker.io/v2/{}/manifests/{}",
|
||||
repository, manifest_digest
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"https://{}/v2/{}/manifests/{}",
|
||||
registry, repository, manifest_digest
|
||||
)
|
||||
};
|
||||
|
||||
let arch_response = client
|
||||
.get(&arch_manifest_url)
|
||||
.header("Accept", "application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json")
|
||||
.header("Authorization", format!("Bearer {}", token))
|
||||
.send()
|
||||
.await
|
||||
.context("Failed to get architecture manifest")?;
|
||||
|
||||
if !arch_response.status().is_success() {
|
||||
return Err(anyhow!(
|
||||
"Failed to get arch manifest: HTTP {}",
|
||||
arch_response.status()
|
||||
));
|
||||
}
|
||||
|
||||
let arch_body = arch_response.text().await?;
|
||||
|
||||
// The arch manifest was requested by digest: verify the response
|
||||
// matches, since the layer digests read from it are trusted next.
|
||||
let mut manifest_hasher = DownloadHasher::new();
|
||||
manifest_hasher.update(arch_body.as_bytes());
|
||||
manifest_hasher
|
||||
.verify(Some(manifest_digest))
|
||||
.context("architecture manifest failed digest verification")?;
|
||||
|
||||
let arch_manifest: Value = serde_json::from_str(&arch_body)
|
||||
.context("Failed to parse architecture manifest JSON")?;
|
||||
|
||||
// Get all layers from architecture manifest (in order)
|
||||
arch_manifest["layers"]
|
||||
.as_array()
|
||||
.ok_or_else(|| anyhow!("No layers in architecture manifest"))?
|
||||
.clone()
|
||||
} else {
|
||||
// Single architecture manifest - get all layers in order
|
||||
manifest["layers"]
|
||||
.as_array()
|
||||
.ok_or_else(|| anyhow!("No layers in manifest"))?
|
||||
.clone()
|
||||
};
|
||||
|
||||
log::debug!("Found {} layers to download", layers.len());
|
||||
|
||||
// Download all layer blobs into a temp directory
|
||||
let temp_dir = tempfile::tempdir()?;
|
||||
|
||||
// layer_names[i] is the filename used both for the downloaded blob and in
|
||||
// layers.manifest. The extension is derived from the layer's mediaType so
|
||||
// extract_oci_layer can dispatch on the filename without relying on
|
||||
// magic-byte detection (which would misfire on a zstd blob named .tar.gz).
|
||||
let mut layer_names: Vec<String> = Vec::with_capacity(layers.len());
|
||||
|
||||
for (i, layer) in layers.iter().enumerate() {
|
||||
let layer_digest = layer["digest"]
|
||||
.as_str()
|
||||
.ok_or_else(|| anyhow!("No digest in layer"))?;
|
||||
|
||||
let media_type = layer["mediaType"].as_str().unwrap_or("");
|
||||
let ext = media_type_to_extension(media_type);
|
||||
let layer_name = format!("layer_{}.{}", i, ext);
|
||||
|
||||
log::debug!(
|
||||
"Fetching layer {}/{}: {}",
|
||||
i + 1,
|
||||
layers.len(),
|
||||
layer_digest
|
||||
);
|
||||
|
||||
let blob_url = if registry == "docker.io" {
|
||||
format!(
|
||||
"https://registry-1.docker.io/v2/{}/blobs/{}",
|
||||
repository, layer_digest
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"https://{}/v2/{}/blobs/{}",
|
||||
registry, repository, layer_digest
|
||||
)
|
||||
};
|
||||
|
||||
let layer_path = temp_dir.path().join(&layer_name);
|
||||
let label = format!("Layer {}/{} ({})", i + 1, layers.len(), layer_name);
|
||||
download_blob_with_auth(
|
||||
client,
|
||||
&blob_url,
|
||||
&layer_path,
|
||||
&token,
|
||||
&label,
|
||||
layer_digest,
|
||||
progress,
|
||||
)
|
||||
.await?;
|
||||
layer_names.push(layer_name);
|
||||
}
|
||||
|
||||
// Write the layer index so extract.rs knows the order
|
||||
{
|
||||
use std::io::Write;
|
||||
let mut manifest_file = std::fs::File::create(temp_dir.path().join("layers.manifest"))?;
|
||||
for name in &layer_names {
|
||||
writeln!(manifest_file, "{}", name)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Bundle layers.manifest + all layer blobs into a single .tar.gz cache file
|
||||
// using the tar + flate2 crates — no external `tar` binary required.
|
||||
let temp_dest = dest.with_extension("tar.partial");
|
||||
|
||||
let bundle_result: anyhow::Result<()> = (|| {
|
||||
use flate2::{write::GzEncoder, Compression};
|
||||
|
||||
let out_file =
|
||||
std::fs::File::create(&temp_dest).context("Failed to create temporary bundle file")?;
|
||||
let mut builder = tar::Builder::new(GzEncoder::new(out_file, Compression::default()));
|
||||
|
||||
builder
|
||||
.append_path_with_name(temp_dir.path().join("layers.manifest"), "layers.manifest")
|
||||
.context("Failed to add layers.manifest to bundle")?;
|
||||
|
||||
for name in &layer_names {
|
||||
builder
|
||||
.append_path_with_name(temp_dir.path().join(name), name)
|
||||
.with_context(|| format!("Failed to add {} to bundle", name))?;
|
||||
}
|
||||
|
||||
// into_inner finalises the tar end-of-archive marker and returns the
|
||||
// GzEncoder; finish() flushes and closes the gzip stream.
|
||||
builder
|
||||
.into_inner()
|
||||
.context("Failed to finalise tar archive")?
|
||||
.finish()
|
||||
.context("Failed to finalise gzip stream")?;
|
||||
|
||||
std::fs::rename(&temp_dest, dest).context("Failed to move bundle to cache destination")
|
||||
})();
|
||||
|
||||
if bundle_result.is_err() {
|
||||
// Best-effort cleanup; ignore errors (file may not exist if creation failed).
|
||||
let _ = std::fs::remove_file(&temp_dest);
|
||||
}
|
||||
bundle_result?;
|
||||
|
||||
Ok(tag_digest)
|
||||
}
|
||||
|
||||
async fn get_auth_token(
|
||||
client: &Client,
|
||||
registry: &str,
|
||||
repository: &str,
|
||||
) -> anyhow::Result<String> {
|
||||
// Docker Hub has a well-known, stable auth endpoint.
|
||||
if registry == "docker.io" {
|
||||
let url = format!(
|
||||
"https://auth.docker.io/token?service=registry.docker.io&scope=repository:{}:pull",
|
||||
repository
|
||||
);
|
||||
log::debug!("Getting auth token from: {}", url);
|
||||
return fetch_bearer_token(client, &url).await;
|
||||
}
|
||||
|
||||
// For every other registry follow the OCI Distribution Spec §4.2:
|
||||
// 1. Probe GET /v2/ unauthenticated.
|
||||
// 2. Read the WWW-Authenticate challenge from the 401 response.
|
||||
// 3. Build the token URL from the advertised realm + service.
|
||||
let probe_url = format!("https://{}/v2/", registry);
|
||||
let probe = client
|
||||
.get(&probe_url)
|
||||
.send()
|
||||
.await
|
||||
.with_context(|| format!("Failed to probe registry at {}", probe_url))?;
|
||||
|
||||
match probe.status().as_u16() {
|
||||
200 => {
|
||||
// Registry requires no authentication (e.g. local insecure registry).
|
||||
return Ok(String::new());
|
||||
}
|
||||
401 => {}
|
||||
other => {
|
||||
// Unexpected status; proceed without a token and let the manifest
|
||||
// request fail with a more descriptive error.
|
||||
log::warn!(
|
||||
"registry probe returned HTTP {}; trying without auth",
|
||||
other
|
||||
);
|
||||
return Ok(String::new());
|
||||
}
|
||||
}
|
||||
|
||||
// Parse the WWW-Authenticate challenge.
|
||||
let www_auth = probe
|
||||
.headers()
|
||||
.get("WWW-Authenticate")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.ok_or_else(|| {
|
||||
anyhow!(
|
||||
"Registry {} returned 401 but no WWW-Authenticate header",
|
||||
registry
|
||||
)
|
||||
})?;
|
||||
|
||||
let (realm, service) = parse_www_authenticate(www_auth)
|
||||
.ok_or_else(|| anyhow!("Could not parse WWW-Authenticate header: {}", www_auth))?;
|
||||
|
||||
// The probe scope is generic; override it with the per-repository pull scope.
|
||||
let token_url = format!(
|
||||
"{}?service={}&scope=repository:{}:pull",
|
||||
realm, service, repository
|
||||
);
|
||||
log::debug!("Getting auth token from: {}", token_url);
|
||||
fetch_bearer_token(client, &token_url).await
|
||||
}
|
||||
|
||||
/// Parse a `Bearer realm="...",service="..."[,scope="..."]` header value.
|
||||
/// Returns `(realm, service)` on success.
|
||||
fn parse_www_authenticate(header: &str) -> Option<(String, String)> {
|
||||
let params = header.strip_prefix("Bearer ")?;
|
||||
|
||||
let mut realm: Option<String> = None;
|
||||
let mut service: Option<String> = None;
|
||||
|
||||
// Split on commas that lie outside quoted strings.
|
||||
let mut start = 0;
|
||||
let mut in_quotes = false;
|
||||
let bytes = params.as_bytes();
|
||||
let mut parts: Vec<&str> = Vec::new();
|
||||
for i in 0..bytes.len() {
|
||||
match bytes[i] {
|
||||
b'"' => in_quotes = !in_quotes,
|
||||
b',' if !in_quotes => {
|
||||
parts.push(params[start..i].trim());
|
||||
start = i + 1;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
parts.push(params[start..].trim());
|
||||
|
||||
for part in parts {
|
||||
if let Some(val) = part.strip_prefix("realm=") {
|
||||
realm = Some(val.trim_matches('"').to_string());
|
||||
} else if let Some(val) = part.strip_prefix("service=") {
|
||||
service = Some(val.trim_matches('"').to_string());
|
||||
}
|
||||
}
|
||||
|
||||
Some((realm?, service?))
|
||||
}
|
||||
|
||||
/// Fetch a bearer token from a fully-constructed token URL.
|
||||
async fn fetch_bearer_token(client: &Client, token_url: &str) -> anyhow::Result<String> {
|
||||
let response = client
|
||||
.get(token_url)
|
||||
.send()
|
||||
.await
|
||||
.context("Failed to request auth token")?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
// Registry may allow anonymous pulls; proceed with an empty token.
|
||||
return Ok(String::new());
|
||||
}
|
||||
|
||||
let body = response.text().await?;
|
||||
let json: Value = serde_json::from_str(&body).context("Failed to parse auth token response")?;
|
||||
|
||||
// OCI spec uses "token"; Docker also emits "access_token".
|
||||
json["token"]
|
||||
.as_str()
|
||||
.or_else(|| json["access_token"].as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| anyhow!("No token field in auth response: {}", body))
|
||||
}
|
||||
|
||||
async fn download_blob_with_auth(
|
||||
client: &Client,
|
||||
url: &str,
|
||||
dest: &Path,
|
||||
token: &str,
|
||||
label: &str,
|
||||
expected_digest: &str,
|
||||
progress: &dyn Progress,
|
||||
) -> anyhow::Result<()> {
|
||||
let mut request = client.get(url);
|
||||
if !token.is_empty() {
|
||||
request = request.header("Authorization", format!("Bearer {}", token));
|
||||
}
|
||||
|
||||
let response = request.send().await.context("Failed to start download")?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(anyhow!(
|
||||
"Failed to download blob: HTTP {}",
|
||||
response.status()
|
||||
));
|
||||
}
|
||||
|
||||
let total_size = response.content_length().unwrap_or(0);
|
||||
|
||||
progress.begin(label, Unit::Bytes, (total_size > 0).then_some(total_size));
|
||||
|
||||
let mut file = tokio::fs::File::create(dest)
|
||||
.await
|
||||
.context("Failed to create destination file")?;
|
||||
|
||||
let mut hasher = DownloadHasher::new();
|
||||
let mut stream = response.bytes_stream();
|
||||
use futures_util::StreamExt;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.context("Failed to read chunk")?;
|
||||
file.write_all(&chunk)
|
||||
.await
|
||||
.context("Failed to write chunk")?;
|
||||
hasher.update(&chunk);
|
||||
progress.advance(chunk.len() as u64);
|
||||
}
|
||||
|
||||
file.flush().await?;
|
||||
progress.finish();
|
||||
|
||||
// The blob was addressed by this digest in the manifest; a mismatch means
|
||||
// truncated or tampered content. The blob lives in a temp directory that
|
||||
// is discarded on error, so a bad layer never reaches the cache.
|
||||
hasher.verify(Some(expected_digest)).with_context(|| {
|
||||
format!(
|
||||
"layer blob {} failed verification against manifest digest",
|
||||
url
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Map an OCI layer mediaType to the appropriate file extension.
|
||||
/// The extension is stored in layers.manifest and used by extract_oci_layer
|
||||
/// for compression dispatch, so it must accurately reflect the blob encoding.
|
||||
fn media_type_to_extension(media_type: &str) -> &'static str {
|
||||
match media_type {
|
||||
// Docker V2 schema 2
|
||||
"application/vnd.docker.image.rootfs.diff.tar.gzip" => "tar.gz",
|
||||
// OCI image spec
|
||||
"application/vnd.oci.image.layer.v1.tar+gzip" => "tar.gz",
|
||||
"application/vnd.oci.image.layer.v1.tar+zstd" => "tar.zst",
|
||||
"application/vnd.oci.image.layer.v1.tar" => "tar",
|
||||
// Non-distributable variants (same encoding, different semantics)
|
||||
"application/vnd.oci.image.layer.nondistributable.v1.tar+gzip" => "tar.gz",
|
||||
"application/vnd.oci.image.layer.nondistributable.v1.tar+zstd" => "tar.zst",
|
||||
"application/vnd.oci.image.layer.nondistributable.v1.tar" => "tar",
|
||||
// Unknown: fall back to gzip (historically the most common format)
|
||||
// and let magic-byte detection in extract_oci_layer handle it.
|
||||
_ => "tar.gz",
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::expected_digest_from_checksums;
|
||||
use super::parse_www_authenticate;
|
||||
use super::DownloadHasher;
|
||||
|
||||
// sha256("abc") and sha512("abc") — standard test vectors
|
||||
const SHA256_ABC: &str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad";
|
||||
const SHA512_ABC: &str = "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a\
|
||||
2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f";
|
||||
|
||||
fn hasher_with(content: &[u8]) -> DownloadHasher {
|
||||
let mut hasher = DownloadHasher::new();
|
||||
hasher.update(content);
|
||||
hasher
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_oci_digests() {
|
||||
let h = hasher_with(b"abc");
|
||||
assert!(h.verify(Some(&format!("sha256:{}", SHA256_ABC))).is_ok());
|
||||
assert!(h.verify(Some(&format!("sha512:{}", SHA512_ABC))).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_bare_hex_is_sha256() {
|
||||
// distro checksum files carry a bare sha256 hex string
|
||||
let h = hasher_with(b"abc");
|
||||
assert!(h.verify(Some(SHA256_ABC)).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_mismatch_errors() {
|
||||
let h = hasher_with(b"abc");
|
||||
let wrong = format!("sha256:{}", "0".repeat(64));
|
||||
assert!(h.verify(Some(&wrong)).is_err());
|
||||
// right hash, wrong algorithm name pinned to it
|
||||
let algo_mismatch = format!("sha512:{}", SHA256_ABC);
|
||||
assert!(h.verify(Some(&algo_mismatch)).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_none_and_unknown_algo_accept() {
|
||||
let h = hasher_with(b"abc");
|
||||
assert!(h.verify(None).is_ok());
|
||||
assert!(h.verify(Some("md5:0123456789abcdef")).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checksum_parsing_bsd_style() {
|
||||
// Alpine's per-file .sha256: "<hex> <filename>"
|
||||
let text = format!("{} alpine-minirootfs-3.23.0-x86_64.tar.gz\n", SHA256_ABC);
|
||||
assert_eq!(
|
||||
expected_digest_from_checksums(&text, "alpine-minirootfs-3.23.0-x86_64.tar.gz"),
|
||||
Some(SHA256_ABC.to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checksum_parsing_binary_marker() {
|
||||
// Ubuntu's SHA256SUMS: "<hex> *<filename>" (binary-mode marker)
|
||||
let text = format!("{} *noble-base-amd64.tar.gz\n", SHA256_ABC);
|
||||
assert_eq!(
|
||||
expected_digest_from_checksums(&text, "noble-base-amd64.tar.gz"),
|
||||
Some(SHA256_ABC.to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checksum_parsing_missing_and_malformed() {
|
||||
let text = format!("{} *other-file.tar.gz\nnot-a-checksum-line\n", SHA256_ABC);
|
||||
assert_eq!(
|
||||
expected_digest_from_checksums(&text, "noble-base-amd64.tar.gz"),
|
||||
None
|
||||
);
|
||||
assert_eq!(expected_digest_from_checksums("", "x"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_docker_hub_challenge() {
|
||||
let hdr = r#"Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:library/ubuntu:pull""#;
|
||||
let (realm, service) = parse_www_authenticate(hdr).unwrap();
|
||||
assert_eq!(realm, "https://auth.docker.io/token");
|
||||
assert_eq!(service, "registry.docker.io");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_quay_challenge() {
|
||||
let hdr = r#"Bearer realm="https://quay.io/v2/auth",service="quay.io",scope="repository:centos/centos:pull""#;
|
||||
let (realm, service) = parse_www_authenticate(hdr).unwrap();
|
||||
assert_eq!(realm, "https://quay.io/v2/auth");
|
||||
assert_eq!(service, "quay.io");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ghcr_challenge() {
|
||||
let hdr = r#"Bearer realm="https://ghcr.io/token",service="ghcr.io",scope="repository:owner/repo:pull""#;
|
||||
let (realm, service) = parse_www_authenticate(hdr).unwrap();
|
||||
assert_eq!(realm, "https://ghcr.io/token");
|
||||
assert_eq!(service, "ghcr.io");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gcr_challenge() {
|
||||
let hdr = r#"Bearer realm="https://gcr.io/v2/token",service="gcr.io""#;
|
||||
let (realm, service) = parse_www_authenticate(hdr).unwrap();
|
||||
assert_eq!(realm, "https://gcr.io/v2/token");
|
||||
assert_eq!(service, "gcr.io");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_value_with_comma_in_scope() {
|
||||
// scope field itself contains no comma but realm/service values may
|
||||
// contain other special chars — ensure the quoted-comma splitter works
|
||||
let hdr = r#"Bearer realm="https://example.com/auth",service="example.com",scope="repository:a/b:pull""#;
|
||||
let (realm, service) = parse_www_authenticate(hdr).unwrap();
|
||||
assert_eq!(realm, "https://example.com/auth");
|
||||
assert_eq!(service, "example.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_not_bearer_returns_none() {
|
||||
assert!(parse_www_authenticate("Basic realm=\"registry\"").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_media_type_to_extension_known_types() {
|
||||
use super::media_type_to_extension;
|
||||
assert_eq!(
|
||||
media_type_to_extension("application/vnd.docker.image.rootfs.diff.tar.gzip"),
|
||||
"tar.gz"
|
||||
);
|
||||
assert_eq!(
|
||||
media_type_to_extension("application/vnd.oci.image.layer.v1.tar+gzip"),
|
||||
"tar.gz"
|
||||
);
|
||||
assert_eq!(
|
||||
media_type_to_extension("application/vnd.oci.image.layer.v1.tar+zstd"),
|
||||
"tar.zst"
|
||||
);
|
||||
assert_eq!(
|
||||
media_type_to_extension("application/vnd.oci.image.layer.v1.tar"),
|
||||
"tar"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_media_type_to_extension_unknown_falls_back_to_gz() {
|
||||
use super::media_type_to_extension;
|
||||
assert_eq!(media_type_to_extension(""), "tar.gz");
|
||||
assert_eq!(media_type_to_extension("text/plain"), "tar.gz");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
//! Typed error for the public library API.
|
||||
//!
|
||||
//! Public functions return [`Result`] carrying [`Error`] so consumers can
|
||||
//! match on the failure modes that call for different handling, a host
|
||||
//! without working KVM, an architecture the registry does not serve,
|
||||
//! versus ordinary failures. Everything without a semantic variant
|
||||
//! surfaces as [`Error::Other`], which preserves the full anyhow context
|
||||
//! chain.
|
||||
|
||||
/// Errors returned by the ecrou library API.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
/// No working KVM on this host, and ecrou has no software-emulation
|
||||
/// fallback. The remediation names the usual causes in order.
|
||||
#[error(
|
||||
"KVM is not available\n\n\
|
||||
ecrou boots microVMs and needs /dev/kvm:\n\
|
||||
\x20 check the module: lsmod | grep kvm\n\
|
||||
\x20 load it: sudo modprobe kvm_intel (or kvm_amd)\n\
|
||||
\x20 nested virt (in a VM): enable it on the hypervisor, then\n\
|
||||
\x20 cat /sys/module/kvm_intel/parameters/nested\n\
|
||||
\x20 access: sudo usermod -aG kvm $USER (re-login)"
|
||||
)]
|
||||
KvmUnavailable,
|
||||
|
||||
/// The registry publishes no linux manifest for the requested
|
||||
/// architecture; `available` lists the architectures it does serve.
|
||||
#[error("No manifest found for architecture '{arch}'. Available: {available}")]
|
||||
ArchNotAvailable { arch: String, available: String },
|
||||
|
||||
/// Any other failure, preserving the full context chain.
|
||||
#[error(transparent)]
|
||||
Other(#[from] anyhow::Error),
|
||||
}
|
||||
|
||||
/// Result alias for the public library API: `Result<T>` carries [`Error`].
|
||||
pub type Result<T, E = Error> = std::result::Result<T, E>;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn kvm_unavailable_renders_remediation() {
|
||||
let msg = Error::KvmUnavailable.to_string();
|
||||
assert!(msg.contains("KVM is not available"));
|
||||
assert!(msg.contains("modprobe kvm_intel"));
|
||||
assert!(msg.contains("usermod -aG kvm"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_preserves_chain_through_sources() {
|
||||
let inner = anyhow::Error::new(std::io::Error::new(
|
||||
std::io::ErrorKind::NotFound,
|
||||
"missing file",
|
||||
))
|
||||
.context("extract failed");
|
||||
let e = Error::Other(inner);
|
||||
// Walk the source chain: the io error must remain reachable for
|
||||
// error-report crates, even though Display shows only the top.
|
||||
let mut source: Option<&(dyn std::error::Error + 'static)> = Some(&e);
|
||||
let mut found = false;
|
||||
while let Some(err) = source {
|
||||
if err.to_string().contains("missing file") {
|
||||
found = true;
|
||||
}
|
||||
source = err.source();
|
||||
}
|
||||
assert!(found, "chain root lost through Error::Other");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,630 @@
|
||||
use crate::progress::{Progress, Unit};
|
||||
use anyhow::Context;
|
||||
|
||||
use crate::Result;
|
||||
use std::fs::File;
|
||||
use std::io::{BufReader, Read};
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
||||
/// Extract a tarball to the specified directory
|
||||
pub fn extract_tarball(tarball: &Path, dest: &Path, progress: &dyn Progress) -> Result<()> {
|
||||
let filename = tarball.file_name().and_then(|n| n.to_str()).unwrap_or("");
|
||||
|
||||
// OCI bundle files are always named "oci-…" by generate_cache_filename.
|
||||
// Use the filename prefix as a zero-I/O dispatch signal instead of
|
||||
// scanning the archive for a layers.manifest entry (which required
|
||||
// reading the entire compressed archive twice for large OCI images).
|
||||
if filename.starts_with("oci-") {
|
||||
log::debug!("Detected multi-layer OCI image, extracting layers...");
|
||||
return Ok(extract_multi_layer_oci(tarball, dest, progress)?);
|
||||
}
|
||||
|
||||
Ok(extract_single_archive(tarball, dest, progress)?)
|
||||
}
|
||||
|
||||
/// Extract a plain rootfs tarball, detecting the compression format from
|
||||
/// the filename with a magic-byte fallback.
|
||||
fn extract_single_archive(
|
||||
tarball: &Path,
|
||||
dest: &Path,
|
||||
progress: &dyn Progress,
|
||||
) -> anyhow::Result<()> {
|
||||
let filename = tarball.file_name().and_then(|n| n.to_str()).unwrap_or("");
|
||||
|
||||
let file = File::open(tarball)
|
||||
.with_context(|| format!("Failed to open tarball: {}", tarball.display()))?;
|
||||
|
||||
let reader = BufReader::new(file);
|
||||
let mut fixups = DeferredFixups::default();
|
||||
|
||||
// Detect compression format from filename
|
||||
if filename.ends_with(".tar.gz") || filename.ends_with(".tgz") {
|
||||
extract_gz(reader, dest, &mut fixups, progress)?;
|
||||
} else if filename.ends_with(".tar.xz") || filename.ends_with(".txz") {
|
||||
extract_xz(reader, dest, &mut fixups, progress)?;
|
||||
} else if filename.ends_with(".tar.zst") || filename.ends_with(".tar.zstd") {
|
||||
extract_zst(reader, dest, &mut fixups, progress)?;
|
||||
} else if filename.ends_with(".tar") {
|
||||
extract_tar(reader, dest, &mut fixups, progress)?;
|
||||
} else {
|
||||
// Try to detect from magic bytes
|
||||
let mut magic = [0u8; 6];
|
||||
let mut peek_reader = BufReader::new(File::open(tarball)?);
|
||||
peek_reader.read_exact(&mut magic)?;
|
||||
|
||||
match magic {
|
||||
[0x1f, 0x8b, ..] => {
|
||||
// gzip magic
|
||||
drop(peek_reader);
|
||||
let file = File::open(tarball)?;
|
||||
extract_gz(BufReader::new(file), dest, &mut fixups, progress)?;
|
||||
}
|
||||
[0xfd, b'7', b'z', b'X', b'Z', 0x00] => {
|
||||
// xz magic
|
||||
drop(peek_reader);
|
||||
let file = File::open(tarball)?;
|
||||
extract_xz(BufReader::new(file), dest, &mut fixups, progress)?;
|
||||
}
|
||||
[0x28, 0xb5, 0x2f, 0xfd, ..] => {
|
||||
// zstd magic
|
||||
drop(peek_reader);
|
||||
let file = File::open(tarball)?;
|
||||
extract_zst(BufReader::new(file), dest, &mut fixups, progress)?;
|
||||
}
|
||||
_ => {
|
||||
// Assume uncompressed tar
|
||||
drop(peek_reader);
|
||||
let file = File::open(tarball)?;
|
||||
extract_tar(BufReader::new(file), dest, &mut fixups, progress)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
apply_fixups(dest, &mut fixups)
|
||||
}
|
||||
|
||||
/// Extract a multi-layer OCI image
|
||||
fn extract_multi_layer_oci(
|
||||
tarball: &Path,
|
||||
dest: &Path,
|
||||
progress: &dyn Progress,
|
||||
) -> anyhow::Result<()> {
|
||||
let filename = tarball.file_name().and_then(|n| n.to_str()).unwrap_or("");
|
||||
|
||||
// Extract the outer OCI bundle (layers.manifest + layer_N.tar.gz files)
|
||||
// into a temp directory. This is our own format, not an OCI layer, so
|
||||
// plain unpack is fine here.
|
||||
let temp_dir = tempfile::tempdir().context("Failed to create temp directory for OCI layers")?;
|
||||
|
||||
let file = File::open(tarball)
|
||||
.with_context(|| format!("Failed to open OCI tarball: {}", tarball.display()))?;
|
||||
let reader = BufReader::new(file);
|
||||
|
||||
if filename.ends_with(".tar.gz") || filename.ends_with(".tgz") {
|
||||
tar::Archive::new(flate2::read::GzDecoder::new(reader))
|
||||
.unpack(temp_dir.path())
|
||||
.context("Failed to unpack OCI bundle")?;
|
||||
} else if filename.ends_with(".tar.xz") || filename.ends_with(".txz") {
|
||||
tar::Archive::new(xz2::read::XzDecoder::new(reader))
|
||||
.unpack(temp_dir.path())
|
||||
.context("Failed to unpack OCI bundle")?;
|
||||
} else if filename.ends_with(".tar.zst") || filename.ends_with(".tar.zstd") {
|
||||
tar::Archive::new(zstd::stream::read::Decoder::new(reader)?)
|
||||
.unpack(temp_dir.path())
|
||||
.context("Failed to unpack OCI bundle")?;
|
||||
} else {
|
||||
tar::Archive::new(reader)
|
||||
.unpack(temp_dir.path())
|
||||
.context("Failed to unpack OCI bundle")?;
|
||||
}
|
||||
|
||||
let manifest_path = temp_dir.path().join("layers.manifest");
|
||||
if !manifest_path.exists() {
|
||||
// Not our layer-bundle layout: a provisioned rootfs persisted over
|
||||
// an OCI cache entry (the dispatch above keys on the "oci-" filename
|
||||
// prefix). The outer archive is a plain rootfs tarball.
|
||||
log::debug!("No layers.manifest in bundle; extracting as plain rootfs tarball");
|
||||
return extract_single_archive(tarball, dest, progress);
|
||||
}
|
||||
|
||||
// Read the layers manifest
|
||||
let manifest =
|
||||
std::fs::read_to_string(&manifest_path).context("Failed to read layers.manifest")?;
|
||||
|
||||
// One fixup set across all layers: later layers routinely write into
|
||||
// directories the base layer ships read-only, and hard-link targets
|
||||
// routinely live in an earlier layer.
|
||||
let mut fixups = DeferredFixups::default();
|
||||
|
||||
// Apply each layer in order with full whiteout handling.
|
||||
for layer_name in manifest.lines() {
|
||||
let layer_path = temp_dir.path().join(layer_name);
|
||||
if !layer_path.exists() {
|
||||
continue;
|
||||
}
|
||||
log::debug!("Extracting layer: {}", layer_name);
|
||||
extract_oci_layer(&layer_path, dest, &mut fixups, progress)?;
|
||||
}
|
||||
|
||||
apply_fixups(dest, &mut fixups)
|
||||
}
|
||||
|
||||
/// Decompress and extract one OCI layer tarball into `dest`, honouring
|
||||
/// whiteout markers. Compression is inferred from the filename then from
|
||||
/// magic bytes.
|
||||
fn extract_oci_layer(
|
||||
layer_path: &Path,
|
||||
dest: &Path,
|
||||
fixups: &mut DeferredFixups,
|
||||
progress: &dyn Progress,
|
||||
) -> anyhow::Result<()> {
|
||||
use std::io::Read;
|
||||
let layer_name = layer_path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("");
|
||||
let file = File::open(layer_path)
|
||||
.with_context(|| format!("Failed to open layer: {}", layer_path.display()))?;
|
||||
let reader = BufReader::new(file);
|
||||
|
||||
// The decompressor type differs per format; erase it behind a trait
|
||||
// object so a single archive-extraction call site serves every branch.
|
||||
let archive: tar::Archive<Box<dyn Read>> =
|
||||
if layer_name.ends_with(".tar.gz") || layer_name.ends_with(".tgz") {
|
||||
tar::Archive::new(Box::new(flate2::read::GzDecoder::new(reader)))
|
||||
} else if layer_name.ends_with(".tar.xz") || layer_name.ends_with(".txz") {
|
||||
tar::Archive::new(Box::new(xz2::read::XzDecoder::new(reader)))
|
||||
} else if layer_name.ends_with(".tar.zst") || layer_name.ends_with(".tar.zstd") {
|
||||
tar::Archive::new(Box::new(zstd::stream::read::Decoder::new(reader)?))
|
||||
} else {
|
||||
// Fall back to magic-byte detection
|
||||
let mut magic = [0u8; 6];
|
||||
let mut peek = BufReader::new(File::open(layer_path)?);
|
||||
let _ = peek.read_exact(&mut magic); // short reads are fine for detection
|
||||
drop(peek);
|
||||
let file = File::open(layer_path)?;
|
||||
match magic {
|
||||
[0x1f, 0x8b, ..] => {
|
||||
tar::Archive::new(Box::new(flate2::read::GzDecoder::new(BufReader::new(file))))
|
||||
}
|
||||
[0xfd, b'7', b'z', b'X', b'Z', 0x00] => {
|
||||
tar::Archive::new(Box::new(xz2::read::XzDecoder::new(BufReader::new(file))))
|
||||
}
|
||||
[0x28, 0xb5, 0x2f, 0xfd, ..] => tar::Archive::new(Box::new(
|
||||
zstd::stream::read::Decoder::new(BufReader::new(file))?,
|
||||
)),
|
||||
_ => tar::Archive::new(Box::new(BufReader::new(file))),
|
||||
}
|
||||
};
|
||||
|
||||
extract_archive(archive, dest, "Extracting OCI layer", fixups, progress)
|
||||
}
|
||||
|
||||
fn extract_gz<R: std::io::Read>(
|
||||
reader: R,
|
||||
dest: &Path,
|
||||
fixups: &mut DeferredFixups,
|
||||
progress: &dyn Progress,
|
||||
) -> anyhow::Result<()> {
|
||||
let gz_decoder = flate2::read::GzDecoder::new(reader);
|
||||
let archive = tar::Archive::new(gz_decoder);
|
||||
|
||||
extract_archive(archive, dest, "Extracting gzip archive", fixups, progress)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn extract_xz<R: std::io::Read>(
|
||||
reader: R,
|
||||
dest: &Path,
|
||||
fixups: &mut DeferredFixups,
|
||||
progress: &dyn Progress,
|
||||
) -> anyhow::Result<()> {
|
||||
let xz_decoder = xz2::read::XzDecoder::new(reader);
|
||||
let archive = tar::Archive::new(xz_decoder);
|
||||
|
||||
extract_archive(archive, dest, "Extracting xz archive", fixups, progress)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn extract_zst<R: std::io::Read>(
|
||||
reader: R,
|
||||
dest: &Path,
|
||||
fixups: &mut DeferredFixups,
|
||||
progress: &dyn Progress,
|
||||
) -> anyhow::Result<()> {
|
||||
let zst_decoder = zstd::Decoder::new(reader)?;
|
||||
let archive = tar::Archive::new(zst_decoder);
|
||||
|
||||
extract_archive(archive, dest, "Extracting zstd archive", fixups, progress)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn extract_tar<R: std::io::Read>(
|
||||
reader: R,
|
||||
dest: &Path,
|
||||
fixups: &mut DeferredFixups,
|
||||
progress: &dyn Progress,
|
||||
) -> anyhow::Result<()> {
|
||||
let archive = tar::Archive::new(reader);
|
||||
|
||||
extract_archive(archive, dest, "Extracting tar archive", fixups, progress)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Archive entries that must not be materialized inline because the tar
|
||||
/// order does not guarantee their prerequisites exist yet.
|
||||
#[derive(Default)]
|
||||
struct DeferredFixups {
|
||||
/// Directory entries with the mode they ship; applied last. Images
|
||||
/// routinely ship read-only directories (Fedora's ca-trust tree is
|
||||
/// dr-xr-xr-x) whose contents would be rejected by an eagerly-applied
|
||||
/// mode.
|
||||
dir_modes: Vec<(PathBuf, u32)>,
|
||||
/// Hard link entries as (destination, raw link target); applied after
|
||||
/// the file pass so targets appearing later in the tar (or in an
|
||||
/// earlier layer) resolve.
|
||||
hard_links: Vec<(PathBuf, PathBuf)>,
|
||||
}
|
||||
|
||||
/// Join an archive-relative entry path onto `dest`, refusing paths that
|
||||
/// traverse above the root (mirrors tar's own unpack component walk).
|
||||
fn safe_join(dest: &Path, rel: &Path) -> Option<PathBuf> {
|
||||
let mut out = dest.to_path_buf();
|
||||
for part in rel.components() {
|
||||
match part {
|
||||
Component::Prefix(_) | Component::RootDir | Component::CurDir => {}
|
||||
Component::ParentDir => return None,
|
||||
Component::Normal(p) => out.push(p),
|
||||
}
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Resolve a raw link target against `dest`, normalizing `..` lexically.
|
||||
/// None when the target climbs out of the rootfs.
|
||||
fn normalize_within(dest: &Path, target: &Path) -> Option<PathBuf> {
|
||||
let mut out = dest.to_path_buf();
|
||||
for part in target.components() {
|
||||
match part {
|
||||
Component::Prefix(_) | Component::RootDir | Component::CurDir => {}
|
||||
Component::ParentDir => {
|
||||
if out == dest {
|
||||
return None;
|
||||
}
|
||||
out.pop();
|
||||
}
|
||||
Component::Normal(p) => out.push(p),
|
||||
}
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Apply hard links and directory modes collected during extraction.
|
||||
fn apply_fixups(dest: &Path, fixups: &mut DeferredFixups) -> anyhow::Result<()> {
|
||||
for (dst, target) in fixups.hard_links.drain(..) {
|
||||
apply_hard_link(dest, &dst, &target)?;
|
||||
}
|
||||
for (path, mode) in fixups.dir_modes.drain(..) {
|
||||
// Errors are ignored: a later layer's whiteout may have removed the
|
||||
// directory, and the mode is cosmetic for our chroot use.
|
||||
let _ = std::fs::set_permissions(&path, std::fs::Permissions::from_mode(mode));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Materialize one hard link. A real hard link is used when the target
|
||||
/// exists on disk; otherwise the entry is preserved verbatim as a symlink —
|
||||
/// some images (Fedora's ca-trust directory-hash) ship links whose targets
|
||||
/// are generated after linking and never exist anywhere in the archive.
|
||||
fn apply_hard_link(dest: &Path, dst: &Path, target: &Path) -> anyhow::Result<()> {
|
||||
if dst.symlink_metadata().is_ok() {
|
||||
std::fs::remove_file(dst)
|
||||
.with_context(|| format!("Failed to replace {}", dst.display()))?;
|
||||
}
|
||||
if let Some(src) = normalize_within(dest, target) {
|
||||
if std::fs::hard_link(&src, dst).is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
std::os::unix::fs::symlink(target, dst)
|
||||
.with_context(|| format!("Failed to link {} -> {}", dst.display(), target.display()))?;
|
||||
log::debug!(
|
||||
"Warning: hard link target {} absent; extracted {} as a symlink",
|
||||
target.display(),
|
||||
dst.display()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Extract tar archive entries, reporting a per-file count through
|
||||
/// `progress` and handling whiteout files for OCI layers
|
||||
fn extract_archive<R: std::io::Read>(
|
||||
mut archive: tar::Archive<R>,
|
||||
dest: &Path,
|
||||
label: &str,
|
||||
fixups: &mut DeferredFixups,
|
||||
progress: &dyn Progress,
|
||||
) -> anyhow::Result<()> {
|
||||
progress.begin(label, Unit::Files, None);
|
||||
|
||||
archive.set_preserve_permissions(true);
|
||||
archive.set_preserve_ownerships(false);
|
||||
archive.set_unpack_xattrs(false);
|
||||
|
||||
let entries = archive
|
||||
.entries()
|
||||
.context("Failed to read archive entries")?;
|
||||
|
||||
let mut count: u64 = 0;
|
||||
for entry in entries {
|
||||
let mut entry = entry.context("Failed to read archive entry")?;
|
||||
|
||||
// Clone the path before any mutable borrow of entry
|
||||
let path = entry.path().context("Invalid tar entry path")?.into_owned();
|
||||
|
||||
let filename = path
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().into_owned())
|
||||
.unwrap_or_default();
|
||||
|
||||
// Handle whiteout files (OCI layer markers for deletions)
|
||||
if filename == ".wh..wh..opq" {
|
||||
// Opaque whiteout: clear all previously-extracted content in the parent directory
|
||||
let parent = path.parent().unwrap_or(Path::new(""));
|
||||
let dest_dir = dest.join(parent);
|
||||
if dest_dir.symlink_metadata().is_ok() {
|
||||
for child in std::fs::read_dir(&dest_dir)
|
||||
.with_context(|| format!("Failed to read {}", dest_dir.display()))?
|
||||
{
|
||||
let child = child?;
|
||||
let child_path = child.path();
|
||||
remove_path(&child_path).with_context(|| {
|
||||
format!("Opaque whiteout: failed to remove {}", child_path.display())
|
||||
})?;
|
||||
}
|
||||
}
|
||||
// Do not extract the .wh..wh..opq marker itself
|
||||
} else if let Some(real_name) = filename.strip_prefix(".wh.") {
|
||||
// Regular whiteout: delete the named path from lower layers
|
||||
let parent = path.parent().unwrap_or(Path::new(""));
|
||||
let target = dest.join(parent).join(real_name);
|
||||
if target.symlink_metadata().is_ok() {
|
||||
remove_path(&target)
|
||||
.with_context(|| format!("Whiteout: failed to remove {}", target.display()))?;
|
||||
}
|
||||
// Do not extract the .wh.* marker itself
|
||||
} else if entry.header().entry_type().is_dir() {
|
||||
// Create the directory but defer its mode: the mode in the tar
|
||||
// may be read-only, and this directory's own contents (and other
|
||||
// layers') still need to be written into it.
|
||||
if let Some(dest_dir) = safe_join(dest, &path) {
|
||||
std::fs::create_dir_all(&dest_dir)
|
||||
.with_context(|| format!("Failed to create {}", dest_dir.display()))?;
|
||||
if let Ok(mode) = entry.header().mode() {
|
||||
fixups.dir_modes.push((dest_dir, mode));
|
||||
}
|
||||
}
|
||||
} else if entry.header().entry_type().is_hard_link() {
|
||||
// Defer: the target may be a later entry in this tar, live in an
|
||||
// earlier layer, or not exist at all (see apply_hard_link).
|
||||
if let Some(dest_path) = safe_join(dest, &path) {
|
||||
if let Some(link) = entry.link_name().context("Invalid hard link target")? {
|
||||
if !link.as_os_str().is_empty() {
|
||||
fixups.hard_links.push((dest_path, link.into_owned()));
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
entry
|
||||
.unpack_in(dest)
|
||||
.with_context(|| format!("Failed to extract {}", path.display()))?;
|
||||
}
|
||||
|
||||
count += 1;
|
||||
progress.advance(1);
|
||||
}
|
||||
|
||||
progress.finish();
|
||||
log::debug!("Extracted {} files", count);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove a path: uses remove_dir_all for real directories, remove_file for
|
||||
/// everything else (regular files, symlinks — including symlinks-to-dirs).
|
||||
fn remove_path(path: &Path) -> std::io::Result<()> {
|
||||
// symlink_metadata does not follow symlinks, so a symlink-to-dir correctly
|
||||
// reports file_type().is_symlink() rather than is_dir().
|
||||
let meta = std::fs::symlink_metadata(path)?;
|
||||
if meta.is_dir() {
|
||||
std::fs::remove_dir_all(path)
|
||||
} else {
|
||||
std::fs::remove_file(path)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::progress::NoProgress;
|
||||
use std::os::unix::fs::{MetadataExt, PermissionsExt};
|
||||
use tar::{Builder, EntryType, Header};
|
||||
|
||||
/// Build a plain tar archive from (path, entry-type, mode, payload)
|
||||
/// tuples in the given order; link payloads are the raw link targets.
|
||||
fn build_tar(path: &Path, entries: &[(&str, EntryType, u32, &[u8])]) {
|
||||
let file = File::create(path).unwrap();
|
||||
let mut ar = Builder::new(file);
|
||||
for (name, kind, mode, payload) in entries {
|
||||
let mut header = Header::new_gnu();
|
||||
header.set_size(if *kind == EntryType::Regular {
|
||||
payload.len() as u64
|
||||
} else {
|
||||
0
|
||||
});
|
||||
header.set_entry_type(*kind);
|
||||
header.set_mode(*mode);
|
||||
if *kind == EntryType::Symlink || *kind == EntryType::Link {
|
||||
header
|
||||
.set_link_name(std::path::PathBuf::from(
|
||||
std::str::from_utf8(payload).unwrap(),
|
||||
))
|
||||
.unwrap();
|
||||
}
|
||||
header.set_cksum();
|
||||
if *kind == EntryType::Regular {
|
||||
ar.append_data(&mut header, name, *payload).unwrap();
|
||||
} else {
|
||||
ar.append_data(&mut header, name, std::io::empty()).unwrap();
|
||||
}
|
||||
}
|
||||
ar.finish().unwrap();
|
||||
}
|
||||
|
||||
/// Fedora's ca-certificates ships etc/pki/ca-trust/extracted/pem/
|
||||
/// directory-hash/ as a read-only (0555) directory full of per-cert hash
|
||||
/// symlinks (002c0b4f.0 -> GlobalSign_Root_R46.pem) whose targets come
|
||||
/// later in the tar or not at all, plus hard links elsewhere in the
|
||||
/// image. Extracting must never open or follow a link payload, and must
|
||||
/// not choke on the read-only directory.
|
||||
#[test]
|
||||
fn fedora_directory_hash_links_extract_without_following() {
|
||||
let src = tempfile::tempdir().unwrap();
|
||||
let out = tempfile::tempdir().unwrap();
|
||||
let tarball = src.path().join("rootfs.tar");
|
||||
build_tar(
|
||||
&tarball,
|
||||
&[
|
||||
(
|
||||
"etc/pki/ca-trust/extracted/pem/directory-hash/",
|
||||
EntryType::Directory,
|
||||
0o555,
|
||||
b"",
|
||||
),
|
||||
// symlink whose target appears later in the archive
|
||||
(
|
||||
"etc/pki/ca-trust/extracted/pem/directory-hash/002c0b4f.0",
|
||||
EntryType::Symlink,
|
||||
0o777,
|
||||
b"GlobalSign_Root_R46.pem",
|
||||
),
|
||||
// symlink whose target never appears (dangling)
|
||||
(
|
||||
"etc/pki/ca-trust/extracted/pem/directory-hash/01419da9.0",
|
||||
EntryType::Symlink,
|
||||
0o777,
|
||||
b"Missing_Root.pem",
|
||||
),
|
||||
// the real cert file, written after the read-only dir exists
|
||||
(
|
||||
"etc/pki/ca-trust/extracted/pem/directory-hash/GlobalSign_Root_R46.pem",
|
||||
EntryType::Regular,
|
||||
0o644,
|
||||
b"-----BEGIN CERTIFICATE-----",
|
||||
),
|
||||
],
|
||||
);
|
||||
|
||||
extract_tarball(&tarball, out.path(), &NoProgress).unwrap();
|
||||
|
||||
let base = out
|
||||
.path()
|
||||
.join("etc/pki/ca-trust/extracted/pem/directory-hash");
|
||||
assert_eq!(
|
||||
std::fs::read_link(base.join("002c0b4f.0")).unwrap(),
|
||||
Path::new("GlobalSign_Root_R46.pem")
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read_link(base.join("01419da9.0")).unwrap(),
|
||||
Path::new("Missing_Root.pem"),
|
||||
"dangling symlinks must be created verbatim, not followed"
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(base.join("GlobalSign_Root_R46.pem")).unwrap(),
|
||||
b"-----BEGIN CERTIFICATE-----"
|
||||
);
|
||||
let dir_mode = base.symlink_metadata().unwrap().permissions().mode() & 0o7777;
|
||||
assert_eq!(dir_mode, 0o555, "shipped directory mode must be restored");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hard_link_to_later_target_extracts() {
|
||||
let src = tempfile::tempdir().unwrap();
|
||||
let out = tempfile::tempdir().unwrap();
|
||||
let tarball = src.path().join("rootfs.tar");
|
||||
build_tar(
|
||||
&tarball,
|
||||
&[
|
||||
("etc/dir/", EntryType::Directory, 0o755, b""),
|
||||
// hard link recorded before its target
|
||||
(
|
||||
"etc/dir/link.0",
|
||||
EntryType::Link,
|
||||
0o644,
|
||||
b"etc/dir/real.pem",
|
||||
),
|
||||
("etc/dir/real.pem", EntryType::Regular, 0o644, b"payload"),
|
||||
],
|
||||
);
|
||||
|
||||
extract_tarball(&tarball, out.path(), &NoProgress).unwrap();
|
||||
assert_eq!(
|
||||
std::fs::read(out.path().join("etc/dir/link.0")).unwrap(),
|
||||
b"payload"
|
||||
);
|
||||
let link = out
|
||||
.path()
|
||||
.join("etc/dir/link.0")
|
||||
.symlink_metadata()
|
||||
.unwrap();
|
||||
assert!(
|
||||
!link.file_type().is_symlink(),
|
||||
"target exists later in the tar, so a real hard link is expected"
|
||||
);
|
||||
let link_ino = std::fs::metadata(out.path().join("etc/dir/link.0"))
|
||||
.unwrap()
|
||||
.ino();
|
||||
let real_ino = std::fs::metadata(out.path().join("etc/dir/real.pem"))
|
||||
.unwrap()
|
||||
.ino();
|
||||
assert_eq!(link_ino, real_ino);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dangling_hard_link_does_not_abort_extraction() {
|
||||
let src = tempfile::tempdir().unwrap();
|
||||
let out = tempfile::tempdir().unwrap();
|
||||
let tarball = src.path().join("rootfs.tar");
|
||||
build_tar(
|
||||
&tarball,
|
||||
&[
|
||||
("etc/dir/", EntryType::Directory, 0o755, b""),
|
||||
// target is not in the archive at all
|
||||
(
|
||||
"etc/dir/dangling.0",
|
||||
EntryType::Link,
|
||||
0o644,
|
||||
b"etc/dir/absent.pem",
|
||||
),
|
||||
("etc/dir/keep.txt", EntryType::Regular, 0o644, b"kept"),
|
||||
],
|
||||
);
|
||||
|
||||
extract_tarball(&tarball, out.path(), &NoProgress).unwrap();
|
||||
assert_eq!(
|
||||
std::fs::read(out.path().join("etc/dir/keep.txt")).unwrap(),
|
||||
b"kept",
|
||||
"extraction must not abort on an unresolvable hard link"
|
||||
);
|
||||
let meta = out.path().join("etc/dir/dangling.0").symlink_metadata();
|
||||
assert!(meta.map(|m| m.file_type().is_symlink()).unwrap_or(false));
|
||||
assert_eq!(
|
||||
std::fs::read_link(out.path().join("etc/dir/dangling.0")).unwrap(),
|
||||
Path::new("etc/dir/absent.pem")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
//! ecrou - one-shot microVM environments on libkrun.
|
||||
//!
|
||||
//! This crate is the library behind the `ecrou` CLI. It resolves distro
|
||||
//! and OCI image references, downloads them through a content cache,
|
||||
//! extracts the rootfs into a scratch directory, and boots it as a
|
||||
//! one-shot microVM: one command runs as the machine's only workload,
|
||||
//! its stdio streams through, its exit code comes back, and the machine
|
||||
//! is thrown away. The rootfs crosses as a virtiofs share, the workspace
|
||||
//! as explicit shares the consumer names.
|
||||
//!
|
||||
//! The founding rule: the host process never mounts a filesystem and
|
||||
//! never enters a namespace. Every boundary crossing is a file or a
|
||||
//! socket, which is what lets ecrou run inside a strict snap's seccomp
|
||||
//! allowlist. See SPEC.md.
|
||||
//!
|
||||
//! The main entry points are [`rootfs`] (cache-aware rootfs preparation
|
||||
//! and persistence) and [`vm`] (running a command inside a prepared
|
||||
//! rootfs). Public functions return [`Result`] carrying the [`Error`]
|
||||
//! enum, so consumers can match on host-readiness failures
|
||||
//! ([`Error::KvmUnavailable`], [`Error::ArchNotAvailable`]) versus
|
||||
//! ordinary failures ([`Error::Other`], which preserves the full context
|
||||
//! chain).
|
||||
|
||||
pub mod config;
|
||||
pub mod distro;
|
||||
pub mod download;
|
||||
pub mod error;
|
||||
pub mod extract;
|
||||
pub mod progress;
|
||||
pub mod rootfs;
|
||||
pub mod utils;
|
||||
|
||||
pub use error::{Error, Result};
|
||||
pub use progress::{NoProgress, Progress, Unit};
|
||||
pub use rootfs::{PrepareRequest, PreparedRootfs, RootfsCache};
|
||||
@@ -0,0 +1,42 @@
|
||||
//! Progress reporting for long-running operations.
|
||||
//!
|
||||
//! The library never touches the terminal: downloads, extraction and
|
||||
//! initramfs creation report their progress through the [`Progress`]
|
||||
//! sink supplied by the caller. Consumers choose what to render —
|
||||
//! progress bars, spinners, structured events — or pass [`NoProgress`]
|
||||
//! for silence.
|
||||
|
||||
/// What a progress operation counts.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Unit {
|
||||
/// Bytes transferred (downloads).
|
||||
Bytes,
|
||||
/// Files processed (extraction, archive creation).
|
||||
Files,
|
||||
}
|
||||
|
||||
/// Sink for progress of long-running library operations.
|
||||
///
|
||||
/// Operations are sequential: `begin` starts the next operation (any
|
||||
/// previous one is implicitly over), `advance` counts completed units,
|
||||
/// `finish` marks its end. Sinks must be callable from any thread.
|
||||
pub trait Progress: Send + Sync {
|
||||
/// An operation labelled `label` starts, counting `unit`s; `total`
|
||||
/// is the unit count when known upfront.
|
||||
fn begin(&self, label: &str, unit: Unit, total: Option<u64>);
|
||||
|
||||
/// `n` more units of the current operation completed.
|
||||
fn advance(&self, n: u64);
|
||||
|
||||
/// The current operation ended.
|
||||
fn finish(&self);
|
||||
}
|
||||
|
||||
/// A [`Progress`] sink that discards every event.
|
||||
pub struct NoProgress;
|
||||
|
||||
impl Progress for NoProgress {
|
||||
fn begin(&self, _: &str, _: Unit, _: Option<u64>) {}
|
||||
fn advance(&self, _: u64) {}
|
||||
fn finish(&self) {}
|
||||
}
|
||||
@@ -0,0 +1,597 @@
|
||||
//! Cache-aware rootfs lifecycle: resolve an image reference, download
|
||||
//! through the tarball cache, extract into a scratch directory, and
|
||||
//! optionally persist a provisioned rootfs back into the cache.
|
||||
//!
|
||||
//! The hot-cell flow is [`RootfsCache::prepare_provisioned`]: provision
|
||||
//! once (package installs, user setup, …), and every subsequent call —
|
||||
//! in this or another process sharing the cache — hits the persisted
|
||||
//! entry and skips both the download and the provisioning step:
|
||||
//!
|
||||
//! ```no_run
|
||||
//! use ecrou::{NoProgress, PrepareRequest, RootfsCache};
|
||||
//!
|
||||
//! # fn main() -> anyhow::Result<()> {
|
||||
//! let cache = RootfsCache::new(RootfsCache::default_dir()?);
|
||||
//! let req = PrepareRequest::new("alpine:3.23");
|
||||
//!
|
||||
//! let rootfs = cache.prepare_provisioned(&req, &NoProgress, |rootfs| {
|
||||
//! // Provision by editing the tree host-side. In-rootfs commands
|
||||
//! // arrive with vm::run, which boots the provisioned rootfs.
|
||||
//! std::fs::write(rootfs.dir().join("etc/motd"), "provisioned by ecrou\n")?;
|
||||
//! Ok(())
|
||||
//! })?;
|
||||
//!
|
||||
//! // Amortized runs hit the provisioned entry directly
|
||||
//! let dir = rootfs.dir().to_path_buf();
|
||||
//! # let _ = dir; Ok(())
|
||||
//! # }
|
||||
//! ```
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::{anyhow, Context};
|
||||
|
||||
use crate::Result;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use crate::distro::{
|
||||
map_arch, parse_image_ref, resolve_distro_url, resolve_distro_version, Distro, ImageSource,
|
||||
};
|
||||
use crate::download::{digest_sidecar, download_image, fetch_oci_digest};
|
||||
use crate::extract::extract_tarball;
|
||||
use crate::progress::Progress;
|
||||
|
||||
/// Sidecar suffix marking a cache entry that holds a provisioned rootfs.
|
||||
const PROVISIONED_SUFFIX: &str = "provisioned";
|
||||
|
||||
/// Sidecar path for a cache entry: `foo.tar.gz` → `foo.tar.gz.<suffix>`.
|
||||
fn sidecar(path: &Path, suffix: &str) -> PathBuf {
|
||||
let mut s = path.as_os_str().to_owned();
|
||||
s.push(".");
|
||||
s.push(suffix);
|
||||
PathBuf::from(s)
|
||||
}
|
||||
|
||||
/// The image tarball cache (default location: `~/.cache/ecrou`).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RootfsCache {
|
||||
dir: PathBuf,
|
||||
}
|
||||
|
||||
impl RootfsCache {
|
||||
pub fn new(dir: impl Into<PathBuf>) -> Self {
|
||||
RootfsCache { dir: dir.into() }
|
||||
}
|
||||
|
||||
/// Default cache directory: `~/.cache/ecrou` (or the platform equivalent).
|
||||
pub fn default_dir() -> Result<PathBuf> {
|
||||
dirs::cache_dir()
|
||||
.map(|p| p.join("ecrou"))
|
||||
.ok_or_else(|| anyhow!("Could not determine cache directory").into())
|
||||
}
|
||||
|
||||
pub fn dir(&self) -> &Path {
|
||||
&self.dir
|
||||
}
|
||||
|
||||
/// Cache entry (tarball path) for an image source and architecture.
|
||||
pub fn tarball_path(&self, source: &ImageSource, arch: &str) -> PathBuf {
|
||||
self.dir.join(generate_cache_filename(source, arch))
|
||||
}
|
||||
|
||||
/// Resolve `req`, download through the cache when needed, and extract
|
||||
/// into a fresh scratch directory (removed when the returned
|
||||
/// [`PreparedRootfs`] is dropped). Download and extraction progress is
|
||||
/// reported through `progress`.
|
||||
pub fn prepare(&self, req: &PrepareRequest, progress: &dyn Progress) -> Result<PreparedRootfs> {
|
||||
let arch = req.resolved_arch();
|
||||
let source = resolve_source(&req.image, &arch)?;
|
||||
let cache_path = self.tarball_path(&source, &arch);
|
||||
|
||||
// OCI images with a floating tag (":latest") need a freshness check:
|
||||
// fetch the current manifest digest from the registry and compare it
|
||||
// against the digest stored from the last download. Only re-pull when
|
||||
// the digest has actually changed. On a network error we fall back to
|
||||
// the cached image with a warning rather than hard-failing.
|
||||
let oci_digest_changed = if cache_path.exists() {
|
||||
if let ImageSource::OciImage {
|
||||
registry,
|
||||
repository,
|
||||
tag,
|
||||
..
|
||||
} = &source
|
||||
{
|
||||
if tag == "latest" {
|
||||
match fetch_oci_digest(registry, repository, tag) {
|
||||
Ok(current) => {
|
||||
let stored = std::fs::read_to_string(digest_sidecar(&cache_path)).ok();
|
||||
stored.as_deref() != Some(current.trim())
|
||||
}
|
||||
Err(e) => {
|
||||
log::warn!("could not check image freshness ({}); using cache", e);
|
||||
false
|
||||
}
|
||||
}
|
||||
} else {
|
||||
false // pinned tags are assumed immutable
|
||||
}
|
||||
} else {
|
||||
false
|
||||
}
|
||||
} else {
|
||||
false // cache absent — download triggered by !cache_path.exists() below
|
||||
};
|
||||
|
||||
// Download when not cached, explicitly requested, or the remote digest
|
||||
// has moved
|
||||
if req.no_cache || !cache_path.exists() || oci_digest_changed {
|
||||
std::fs::create_dir_all(&self.dir).with_context(|| {
|
||||
format!("Failed to create cache directory {}", self.dir.display())
|
||||
})?;
|
||||
download_image(&source, &cache_path, &arch, progress)?;
|
||||
} else {
|
||||
log::debug!("Using cached tarball: {}", cache_path.display());
|
||||
}
|
||||
|
||||
Ok(extract_prepared(cache_path, arch, progress)?)
|
||||
}
|
||||
|
||||
/// Prepare a provisioned rootfs with cache amortization: when the cache
|
||||
/// entry is already marked provisioned it is extracted as-is; otherwise
|
||||
/// the rootfs is prepared from the pristine image, `provision` runs
|
||||
/// once, and the result is persisted into the cache so subsequent calls
|
||||
/// (including from other processes sharing this cache) skip both the
|
||||
/// download and the provisioning step.
|
||||
///
|
||||
/// A provisioned entry no longer tracks the upstream image: for
|
||||
/// `:latest` OCI images the freshness check is skipped on a marker hit.
|
||||
/// Re-provision by removing the entry and its `.provisioned` sidecar, or
|
||||
/// by passing `no_cache: true` (which re-downloads, re-provisions and
|
||||
/// re-persists).
|
||||
pub fn prepare_provisioned<F>(
|
||||
&self,
|
||||
req: &PrepareRequest,
|
||||
progress: &dyn Progress,
|
||||
provision: F,
|
||||
) -> Result<PreparedRootfs>
|
||||
where
|
||||
F: FnOnce(&PreparedRootfs) -> anyhow::Result<()>,
|
||||
{
|
||||
let arch = req.resolved_arch();
|
||||
let source = resolve_source(&req.image, &arch)?;
|
||||
let cache_path = self.tarball_path(&source, &arch);
|
||||
let marker = sidecar(&cache_path, PROVISIONED_SUFFIX);
|
||||
|
||||
if !req.no_cache && cache_path.exists() && marker.exists() {
|
||||
log::debug!("Using provisioned cache entry: {}", cache_path.display());
|
||||
return Ok(extract_prepared(cache_path, arch, progress)?);
|
||||
}
|
||||
|
||||
let rootfs = self.prepare(req, progress)?;
|
||||
provision(&rootfs)?;
|
||||
rootfs.persist().with_context(|| {
|
||||
format!(
|
||||
"Failed to persist provisioned rootfs to {}",
|
||||
cache_path.display()
|
||||
)
|
||||
})?;
|
||||
Ok(rootfs)
|
||||
}
|
||||
}
|
||||
|
||||
/// Request for [`RootfsCache::prepare`] / [`RootfsCache::prepare_provisioned`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PrepareRequest {
|
||||
/// Image reference: a distro name ("ubuntu", "ubuntu:noble",
|
||||
/// "alpine:3.23") or an OCI reference ("debian",
|
||||
/// "ghcr.io/org/image:tag", "docker://…").
|
||||
pub image: String,
|
||||
|
||||
/// Target architecture (e.g. "amd64", "arm64"). Empty selects the host
|
||||
/// architecture.
|
||||
pub arch: String,
|
||||
|
||||
/// Ignore the cache and download a fresh image.
|
||||
pub no_cache: bool,
|
||||
}
|
||||
|
||||
impl PrepareRequest {
|
||||
pub fn new(image: impl Into<String>) -> Self {
|
||||
PrepareRequest {
|
||||
image: image.into(),
|
||||
arch: String::new(),
|
||||
no_cache: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The effective architecture: the request value, or the host arch.
|
||||
pub fn resolved_arch(&self) -> String {
|
||||
if self.arch.is_empty() {
|
||||
crate::utils::get_host_arch().debian_name().to_string()
|
||||
} else {
|
||||
self.arch.clone()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A rootfs extracted into a scratch directory, ready for
|
||||
/// [`crate::vm::run`] to boot. The directory is removed when this value is
|
||||
/// dropped, unless [`PreparedRootfs::persist`] ran first.
|
||||
#[derive(Debug)]
|
||||
pub struct PreparedRootfs {
|
||||
rootfs: TempDir,
|
||||
cache_path: PathBuf,
|
||||
arch: String,
|
||||
}
|
||||
|
||||
impl PreparedRootfs {
|
||||
/// The extracted rootfs directory.
|
||||
pub fn dir(&self) -> &Path {
|
||||
self.rootfs.path()
|
||||
}
|
||||
|
||||
/// Architecture this rootfs was prepared for.
|
||||
pub fn arch(&self) -> &str {
|
||||
&self.arch
|
||||
}
|
||||
|
||||
/// The cache entry this rootfs was prepared from.
|
||||
pub fn cache_entry(&self) -> &Path {
|
||||
&self.cache_path
|
||||
}
|
||||
|
||||
/// Persist the current contents of the rootfs into the cache entry it
|
||||
/// was prepared from, replacing the pristine image tarball, and mark
|
||||
/// the entry provisioned. Later preparations for the same image and
|
||||
/// architecture hit this entry directly; note that plain
|
||||
/// [`RootfsCache::prepare`] also returns the provisioned contents
|
||||
/// (indistinguishable from a pristine hit) — only
|
||||
/// [`RootfsCache::prepare_provisioned`] interprets the marker.
|
||||
pub fn persist(&self) -> Result<()> {
|
||||
pack_rootfs(self.dir(), &self.cache_path)?;
|
||||
std::fs::write(sidecar(&self.cache_path, PROVISIONED_SUFFIX), b"").with_context(|| {
|
||||
format!(
|
||||
"Failed to write provisioned marker for {}",
|
||||
self.cache_path.display()
|
||||
)
|
||||
})?;
|
||||
log::debug!("Persisted rootfs to cache: {}", self.cache_path.display());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse an image reference and resolve floating version aliases ("latest",
|
||||
/// "lts", "edge", …) to a concrete version, so the cache key is stable
|
||||
/// (e.g. "ubuntu-noble-amd64", not "ubuntu-latest-amd64") and a future
|
||||
/// release automatically gets its own cache entry.
|
||||
fn resolve_source(image: &str, arch: &str) -> anyhow::Result<ImageSource> {
|
||||
let source = parse_image_ref(image, arch)?;
|
||||
match source {
|
||||
ImageSource::DirectTarball { distro, version } => {
|
||||
let resolved = resolve_distro_version(&distro, version.as_deref(), arch)?;
|
||||
Ok(ImageSource::DirectTarball {
|
||||
distro,
|
||||
version: Some(resolved),
|
||||
})
|
||||
}
|
||||
other => Ok(other),
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_prepared(
|
||||
cache_path: PathBuf,
|
||||
arch: String,
|
||||
progress: &dyn Progress,
|
||||
) -> anyhow::Result<PreparedRootfs> {
|
||||
let temp = tempfile::tempdir().context("Failed to create temporary directory")?;
|
||||
let rootfs = temp.path().to_path_buf();
|
||||
|
||||
log::debug!("Extracting to: {}", rootfs.display());
|
||||
extract_tarball(&cache_path, &rootfs, progress)?;
|
||||
|
||||
Ok(PreparedRootfs {
|
||||
rootfs: temp,
|
||||
cache_path,
|
||||
arch,
|
||||
})
|
||||
}
|
||||
|
||||
/// Write a tarball of `rootfs` at `dest`, compressed according to dest's
|
||||
/// extension (.tar.gz, .tar.xz, .tar.zst, plain .tar), atomically via a
|
||||
/// `.partial` file.
|
||||
fn pack_rootfs(rootfs: &Path, dest: &Path) -> anyhow::Result<()> {
|
||||
// Packing the tree into a tarball placed inside it would recurse
|
||||
// forever: the walker keeps seeing the archive grow.
|
||||
let rootfs_canonical = rootfs
|
||||
.canonicalize()
|
||||
.with_context(|| format!("Failed to resolve rootfs directory {}", rootfs.display()))?;
|
||||
if let Some(parent) = dest.parent() {
|
||||
if let Ok(parent_canonical) = parent.canonicalize() {
|
||||
if parent_canonical.starts_with(&rootfs_canonical) {
|
||||
return Err(anyhow!(
|
||||
"Cache entry {} must live outside the rootfs",
|
||||
dest.display()
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let filename = dest.file_name().and_then(|n| n.to_str()).unwrap_or("");
|
||||
let partial = dest.with_extension("partial");
|
||||
|
||||
let file = std::fs::File::create(&partial)
|
||||
.with_context(|| format!("Failed to create {}", partial.display()))?;
|
||||
|
||||
let result: anyhow::Result<()> = (|| {
|
||||
if filename.ends_with(".tar.gz") || filename.ends_with(".tgz") {
|
||||
let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::default());
|
||||
let encoder = write_rootfs_tar(rootfs, encoder)?;
|
||||
encoder.finish().context("Failed to finalise gzip stream")?;
|
||||
} else if filename.ends_with(".tar.xz") || filename.ends_with(".txz") {
|
||||
let encoder = xz2::write::XzEncoder::new(file, 6);
|
||||
let encoder = write_rootfs_tar(rootfs, encoder)?;
|
||||
encoder.finish().context("Failed to finalise xz stream")?;
|
||||
} else if filename.ends_with(".tar.zst") || filename.ends_with(".tar.zstd") {
|
||||
let encoder = zstd::stream::write::Encoder::new(file, 0)
|
||||
.context("Failed to create zstd encoder")?;
|
||||
let encoder = write_rootfs_tar(rootfs, encoder)?;
|
||||
encoder.finish().context("Failed to finalise zstd stream")?;
|
||||
} else {
|
||||
write_rootfs_tar(rootfs, file)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
|
||||
if result.is_err() {
|
||||
// Best-effort cleanup; ignore errors (file may not exist if creation failed).
|
||||
let _ = std::fs::remove_file(&partial);
|
||||
return result;
|
||||
}
|
||||
|
||||
std::fs::rename(&partial, dest)
|
||||
.with_context(|| format!("Failed to move packed rootfs to {}", dest.display()))
|
||||
}
|
||||
|
||||
/// Write the rootfs tree into a tar archive streamed to `writer`, returning
|
||||
/// the writer so the caller can finalise the compression stream.
|
||||
fn write_rootfs_tar<W: std::io::Write>(rootfs: &Path, writer: W) -> anyhow::Result<W> {
|
||||
let mut builder = tar::Builder::new(writer);
|
||||
// Store symlinks as symlinks; following them would duplicate the target's
|
||||
// contents into the archive (and recurse into symlinked directories).
|
||||
builder.follow_symlinks(false);
|
||||
|
||||
let entries = std::fs::read_dir(rootfs)
|
||||
.with_context(|| format!("Failed to read {}", rootfs.display()))?;
|
||||
|
||||
for entry in entries {
|
||||
let entry =
|
||||
entry.with_context(|| format!("Failed to read entry in {}", rootfs.display()))?;
|
||||
let path = entry.path();
|
||||
let name = entry.file_name();
|
||||
|
||||
// symlink_metadata: never follow symlinks — a top-level symlink to a
|
||||
// directory must be stored as a link, not recursed into.
|
||||
let meta = std::fs::symlink_metadata(&path)
|
||||
.with_context(|| format!("Failed to stat {}", path.display()))?;
|
||||
|
||||
if meta.is_dir() {
|
||||
builder
|
||||
.append_dir_all(&name, &path)
|
||||
.with_context(|| format!("Failed to archive {}", path.display()))?;
|
||||
} else {
|
||||
builder
|
||||
.append_path_with_name(&path, &name)
|
||||
.with_context(|| format!("Failed to archive {}", path.display()))?;
|
||||
}
|
||||
}
|
||||
|
||||
builder
|
||||
.into_inner()
|
||||
.context("Failed to finalise tar archive")
|
||||
}
|
||||
|
||||
/// Generate a cache filename based on the image source
|
||||
fn generate_cache_filename(source: &ImageSource, arch: &str) -> String {
|
||||
match source {
|
||||
ImageSource::DirectTarball { distro, version } => {
|
||||
let distro_name = match distro {
|
||||
Distro::Ubuntu => "ubuntu",
|
||||
Distro::Alpine => "alpine",
|
||||
};
|
||||
let distro_arch = map_arch(*distro, arch);
|
||||
// Get extension from URL
|
||||
let url = resolve_distro_url(distro, version.as_deref(), arch).unwrap_or_default();
|
||||
let ext = get_tarball_extension(&url);
|
||||
format!(
|
||||
"{}-{}-{}.{}",
|
||||
distro_name,
|
||||
version.as_deref().unwrap_or("latest"),
|
||||
distro_arch,
|
||||
ext
|
||||
)
|
||||
}
|
||||
ImageSource::OciImage {
|
||||
registry,
|
||||
repository,
|
||||
tag,
|
||||
architecture,
|
||||
} => {
|
||||
// Sanitize for filename
|
||||
let safe_registry = registry.replace(['.', ':'], "_");
|
||||
let safe_repo = repository.replace(['/', ':'], "_");
|
||||
format!(
|
||||
"oci-{}-{}-{}-{}.tar.gz",
|
||||
safe_registry, safe_repo, tag, architecture
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_tarball_extension(url: &str) -> &str {
|
||||
// Extract extension from URL (e.g., .tar.gz, .tar.xz, .tar.zst)
|
||||
if url.ends_with(".tar.zst") {
|
||||
"tar.zst"
|
||||
} else if url.ends_with(".tar.xz") {
|
||||
"tar.xz"
|
||||
} else if url.ends_with(".tar.gz") {
|
||||
"tar.gz"
|
||||
} else if url.ends_with(".tar.bz2") {
|
||||
"tar.bz2"
|
||||
} else {
|
||||
"tar.gz" // default
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::progress::NoProgress;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
fn direct_tarball(distro: Distro, version: &str) -> ImageSource {
|
||||
ImageSource::DirectTarball {
|
||||
distro,
|
||||
version: Some(version.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_filename_direct_tarball() {
|
||||
// Explicit codename/version: no network lookups
|
||||
let cache = RootfsCache::new("/cache");
|
||||
assert_eq!(
|
||||
cache.tarball_path(&direct_tarball(Distro::Ubuntu, "noble"), "amd64"),
|
||||
PathBuf::from("/cache/ubuntu-noble-amd64.tar.gz")
|
||||
);
|
||||
assert_eq!(
|
||||
cache.tarball_path(&direct_tarball(Distro::Alpine, "3.23.1"), "amd64"),
|
||||
PathBuf::from("/cache/alpine-3.23.1-x86_64.tar.gz")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_filename_oci_sanitized() {
|
||||
let cache = RootfsCache::new("/cache");
|
||||
let source = ImageSource::OciImage {
|
||||
registry: "docker.io".to_string(),
|
||||
repository: "library/ubuntu".to_string(),
|
||||
tag: "latest".to_string(),
|
||||
architecture: "amd64".to_string(),
|
||||
};
|
||||
assert_eq!(
|
||||
cache.tarball_path(&source, "amd64"),
|
||||
PathBuf::from("/cache/oci-docker_io-library_ubuntu-latest-amd64.tar.gz")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolved_arch_defaults_to_host() {
|
||||
let req = PrepareRequest::new("alpine");
|
||||
assert_eq!(
|
||||
req.resolved_arch(),
|
||||
crate::utils::get_host_arch().debian_name()
|
||||
);
|
||||
|
||||
let mut req = PrepareRequest::new("alpine");
|
||||
req.arch = "arm64".to_string();
|
||||
assert_eq!(req.resolved_arch(), "arm64");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sidecar_appends_suffix() {
|
||||
assert_eq!(
|
||||
sidecar(Path::new("/cache/foo.tar.gz"), PROVISIONED_SUFFIX),
|
||||
PathBuf::from("/cache/foo.tar.gz.provisioned")
|
||||
);
|
||||
}
|
||||
|
||||
/// Build a minimal rootfs tree with a nested executable and a symlink,
|
||||
/// pack it, and extract it back through the public extract_tarball path.
|
||||
fn roundtrip(dest_name: &str) {
|
||||
let src = tempfile::tempdir().unwrap();
|
||||
std::fs::write(src.path().join("hello.txt"), b"hello").unwrap();
|
||||
std::fs::create_dir_all(src.path().join("usr/bin")).unwrap();
|
||||
std::fs::write(src.path().join("usr/bin/tool"), b"#!/bin/sh\n").unwrap();
|
||||
std::fs::set_permissions(
|
||||
src.path().join("usr/bin/tool"),
|
||||
std::fs::Permissions::from_mode(0o755),
|
||||
)
|
||||
.unwrap();
|
||||
std::os::unix::fs::symlink("hello.txt", src.path().join("link.txt")).unwrap();
|
||||
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
let dest = cache.path().join(dest_name);
|
||||
pack_rootfs(src.path(), &dest).unwrap();
|
||||
|
||||
let out = tempfile::tempdir().unwrap();
|
||||
crate::extract::extract_tarball(&dest, out.path(), &NoProgress).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
std::fs::read(out.path().join("hello.txt")).unwrap(),
|
||||
b"hello"
|
||||
);
|
||||
let mode = std::fs::metadata(out.path().join("usr/bin/tool"))
|
||||
.unwrap()
|
||||
.permissions()
|
||||
.mode();
|
||||
assert_eq!(mode & 0o777, 0o755, "executable bit must survive");
|
||||
assert_eq!(
|
||||
std::fs::read_link(out.path().join("link.txt")).unwrap(),
|
||||
Path::new("hello.txt")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persist_roundtrip_gzip() {
|
||||
roundtrip("rootfs.tar.gz");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persist_roundtrip_xz() {
|
||||
roundtrip("rootfs.tar.xz");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persist_roundtrip_zstd() {
|
||||
roundtrip("rootfs.tar.zst");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persist_roundtrip_plain_tar() {
|
||||
roundtrip("rootfs.tar");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn persist_writes_entry_and_provisioned_marker() {
|
||||
let rootfs_dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(rootfs_dir.path().join("file.txt"), b"data").unwrap();
|
||||
|
||||
let cache_dir = tempfile::tempdir().unwrap();
|
||||
let cache_path = cache_dir.path().join("ubuntu-noble-amd64.tar.gz");
|
||||
|
||||
let prepared = PreparedRootfs {
|
||||
rootfs: rootfs_dir,
|
||||
cache_path: cache_path.clone(),
|
||||
arch: "amd64".to_string(),
|
||||
};
|
||||
prepared.persist().unwrap();
|
||||
|
||||
assert!(cache_path.exists());
|
||||
assert!(sidecar(&cache_path, PROVISIONED_SUFFIX).exists());
|
||||
|
||||
// The persisted entry extracts back to the original contents
|
||||
let out = tempfile::tempdir().unwrap();
|
||||
crate::extract::extract_tarball(&cache_path, out.path(), &NoProgress).unwrap();
|
||||
assert_eq!(std::fs::read(out.path().join("file.txt")).unwrap(), b"data");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pack_failure_leaves_no_partial_file() {
|
||||
// A nonexistent rootfs must fail cleanly and clean up the .partial
|
||||
let cache = tempfile::tempdir().unwrap();
|
||||
let dest = cache.path().join("x.tar.gz");
|
||||
assert!(pack_rootfs(Path::new("/nonexistent-rootfs"), &dest).is_err());
|
||||
assert!(!dest.exists());
|
||||
assert!(!dest.with_extension("partial").exists());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
use anyhow::anyhow;
|
||||
|
||||
use crate::Result;
|
||||
use std::path::Path;
|
||||
|
||||
// ============================================================================
|
||||
// Constants
|
||||
// ============================================================================
|
||||
|
||||
/// Length of the random hostname suffix (36⁶ ≈ 2×10⁹ distinct values)
|
||||
pub const HOSTNAME_SUFFIX_LEN: usize = 6;
|
||||
|
||||
/// Random lowercase-alphanumeric suffix for hostnames. Seeded from the
|
||||
/// current time and the PID so calls in rapid succession — including from
|
||||
/// separate processes — produce different values; time alone truncated to
|
||||
/// the suffix space repeats within a tight loop.
|
||||
pub fn random_suffix(len: usize) -> String {
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::hash::{Hash, Hasher};
|
||||
|
||||
let mut hasher = DefaultHasher::new();
|
||||
std::time::SystemTime::now().hash(&mut hasher);
|
||||
std::process::id().hash(&mut hasher);
|
||||
let mut state = hasher.finish();
|
||||
|
||||
const CHARS: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789";
|
||||
// Knuth multiplicative LCG — each step advances the full 64-bit state.
|
||||
(0..len)
|
||||
.map(|_| {
|
||||
state = state
|
||||
.wrapping_mul(6364136223846793005)
|
||||
.wrapping_add(1442695040888963407);
|
||||
CHARS[(state >> 33) as usize % CHARS.len()] as char
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Architecture handling
|
||||
// ============================================================================
|
||||
|
||||
/// Architecture representation in different naming conventions
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Arch {
|
||||
/// x86-64 (AMD64, x86_64)
|
||||
Amd64,
|
||||
/// ARM 64-bit (AArch64)
|
||||
Arm64,
|
||||
/// ARM 32-bit hard-float
|
||||
Armhf,
|
||||
/// RISC-V 64-bit
|
||||
Riscv64,
|
||||
/// PowerPC 64-bit little-endian
|
||||
Ppc64el,
|
||||
/// IBM s390x
|
||||
S390x,
|
||||
/// Unknown architecture
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl Arch {
|
||||
/// Get the architecture from a string (any common naming convention)
|
||||
/// Not a FromStr impl: unrecognized names map to Arch::Unknown rather
|
||||
/// than an error, by design.
|
||||
#[allow(clippy::should_implement_trait)]
|
||||
pub fn from_str(s: &str) -> Self {
|
||||
match s {
|
||||
"amd64" | "x86_64" | "x64" => Arch::Amd64,
|
||||
"arm64" | "aarch64" | "arm64v8" => Arch::Arm64,
|
||||
"armhf" | "armv7" | "armv7l" | "arm" => Arch::Armhf,
|
||||
"riscv64" => Arch::Riscv64,
|
||||
"ppc64el" | "ppc64le" => Arch::Ppc64el,
|
||||
"s390x" => Arch::S390x,
|
||||
_ => Arch::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the Debian/Ubuntu style name
|
||||
pub fn debian_name(&self) -> &'static str {
|
||||
match self {
|
||||
Arch::Amd64 => "amd64",
|
||||
Arch::Arm64 => "arm64",
|
||||
Arch::Armhf => "armhf",
|
||||
Arch::Riscv64 => "riscv64",
|
||||
Arch::Ppc64el => "ppc64el",
|
||||
Arch::S390x => "s390x",
|
||||
Arch::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the OCI/Docker registry style name
|
||||
pub fn oci_name(&self) -> &'static str {
|
||||
match self {
|
||||
Arch::Amd64 => "amd64",
|
||||
Arch::Arm64 => "arm64",
|
||||
// OCI uses "arm" for 32-bit ARM with variant field
|
||||
Arch::Armhf => "arm",
|
||||
Arch::Riscv64 => "riscv64",
|
||||
Arch::Ppc64el => "ppc64le",
|
||||
Arch::S390x => "s390x",
|
||||
Arch::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the Alpine style name
|
||||
pub fn alpine_name(&self) -> &'static str {
|
||||
match self {
|
||||
Arch::Amd64 => "x86_64",
|
||||
Arch::Arm64 => "aarch64",
|
||||
Arch::Armhf => "armv7",
|
||||
Arch::Riscv64 => "riscv64",
|
||||
Arch::Ppc64el => "ppc64le",
|
||||
Arch::S390x => "s390x",
|
||||
Arch::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the QEMU binary suffix (e.g., "qemu-system-x86_64")
|
||||
pub fn qemu_system_name(&self) -> &'static str {
|
||||
match self {
|
||||
Arch::Amd64 => "x86_64",
|
||||
Arch::Arm64 => "aarch64",
|
||||
Arch::Armhf => "arm",
|
||||
Arch::Riscv64 => "riscv64",
|
||||
Arch::Ppc64el => "ppc64",
|
||||
Arch::S390x => "s390x",
|
||||
Arch::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the QEMU binfmt_misc name
|
||||
pub fn qemu_binfmt_name(&self) -> &'static str {
|
||||
match self {
|
||||
Arch::Amd64 => "x86_64",
|
||||
Arch::Arm64 => "aarch64",
|
||||
Arch::Armhf => "arm",
|
||||
Arch::Riscv64 => "riscv64",
|
||||
Arch::Ppc64el => "ppc64le",
|
||||
Arch::S390x => "s390x",
|
||||
Arch::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the package suffix for QEMU system emulator
|
||||
pub fn qemu_package_suffix(&self) -> &'static str {
|
||||
match self {
|
||||
Arch::Amd64 => "x86",
|
||||
Arch::Arm64 => "aarch64",
|
||||
Arch::Armhf => "arm",
|
||||
Arch::Riscv64 => "riscv64",
|
||||
Arch::Ppc64el => "ppc",
|
||||
Arch::S390x => "s390x",
|
||||
Arch::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the host system architecture using uname(2) syscall
|
||||
/// This returns the runtime machine string, which is correct even when
|
||||
/// the binary itself is running under emulation.
|
||||
pub fn get_host_arch() -> Arch {
|
||||
// Safety: utsname is a valid zeroed libc struct; uname(2) writes only
|
||||
// within it and NUL-terminates each field before we read machine.
|
||||
let mut utsname: libc::utsname = unsafe { std::mem::zeroed() };
|
||||
let rc = unsafe { libc::uname(&mut utsname) };
|
||||
if rc != 0 {
|
||||
panic!("uname(2) failed: {}", std::io::Error::last_os_error());
|
||||
}
|
||||
let machine = unsafe { std::ffi::CStr::from_ptr(utsname.machine.as_ptr()) };
|
||||
Arch::from_str(machine.to_string_lossy().as_ref())
|
||||
}
|
||||
|
||||
/// Map ecrou architecture names to distro-specific names
|
||||
pub fn map_arch_for_distro(distro: &str, arch: &str) -> String {
|
||||
let arch_enum = Arch::from_str(arch);
|
||||
match distro.to_lowercase().as_str() {
|
||||
"ubuntu" => arch_enum.debian_name().to_string(),
|
||||
"alpine" => arch_enum.alpine_name().to_string(),
|
||||
_ => arch_enum.oci_name().to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Map architecture to OCI registry standard names
|
||||
pub fn map_oci_arch(arch: &str) -> String {
|
||||
Arch::from_str(arch).oci_name().to_string()
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Shell detection
|
||||
// ============================================================================
|
||||
|
||||
/// Detect the best available shell in a rootfs
|
||||
/// Checks for bash first, falls back to sh
|
||||
/// Returns the path relative to the rootfs (e.g., "/bin/bash")
|
||||
pub fn detect_shell(rootfs: &Path) -> &'static str {
|
||||
// Check for bash first (preferred)
|
||||
if rootfs.join("bin/bash").exists() {
|
||||
"/bin/bash"
|
||||
} else if rootfs.join("bin/sh").exists() {
|
||||
"/bin/sh"
|
||||
} else if rootfs.join("usr/bin/bash").exists() {
|
||||
"/usr/bin/bash"
|
||||
} else if rootfs.join("usr/bin/sh").exists() {
|
||||
"/usr/bin/sh"
|
||||
} else {
|
||||
"/bin/sh" // Will fail with clear error if not present
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Memory string validation
|
||||
// ============================================================================
|
||||
|
||||
/// Validate a QEMU memory size string (e.g., "512M", "2G")
|
||||
/// Returns an error if the format is invalid
|
||||
pub fn validate_memory_string(s: &str) -> Result<()> {
|
||||
if s.is_empty() {
|
||||
return Err(anyhow!("Memory size cannot be empty").into());
|
||||
}
|
||||
|
||||
// Must end with a valid suffix or be a plain number
|
||||
let suffix = s.chars().last().unwrap();
|
||||
let has_suffix = suffix.is_ascii_alphabetic();
|
||||
|
||||
let numeric_part = if has_suffix { &s[..s.len() - 1] } else { s };
|
||||
|
||||
// Check for negative numbers
|
||||
if numeric_part.starts_with('-') {
|
||||
return Err(anyhow!("Memory size cannot be negative: {}", s).into());
|
||||
}
|
||||
|
||||
// Must be a valid positive number
|
||||
if numeric_part.is_empty() {
|
||||
return Err(anyhow!("Memory size must have a numeric value: {}", s).into());
|
||||
}
|
||||
|
||||
// Check if it's a valid integer
|
||||
if !numeric_part.chars().all(|c| c.is_ascii_digit()) {
|
||||
return Err(anyhow!(
|
||||
"Memory size must be a positive integer with optional suffix: {}",
|
||||
s
|
||||
)
|
||||
.into());
|
||||
}
|
||||
|
||||
// Check suffix is valid
|
||||
if has_suffix {
|
||||
let valid_suffixes = ['K', 'M', 'G', 'T'];
|
||||
let suffix_upper = suffix.to_ascii_uppercase();
|
||||
if !valid_suffixes.contains(&suffix_upper) {
|
||||
return Err(anyhow!(
|
||||
"Invalid memory suffix '{}'. Valid suffixes: K, M, G, T",
|
||||
suffix
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_arch_from_str() {
|
||||
assert_eq!(Arch::from_str("amd64"), Arch::Amd64);
|
||||
assert_eq!(Arch::from_str("x86_64"), Arch::Amd64);
|
||||
assert_eq!(Arch::from_str("arm64"), Arch::Arm64);
|
||||
assert_eq!(Arch::from_str("aarch64"), Arch::Arm64);
|
||||
assert_eq!(Arch::from_str("armhf"), Arch::Armhf);
|
||||
assert_eq!(Arch::from_str("armv7"), Arch::Armhf);
|
||||
assert_eq!(Arch::from_str("riscv64"), Arch::Riscv64);
|
||||
assert_eq!(Arch::from_str("ppc64el"), Arch::Ppc64el);
|
||||
assert_eq!(Arch::from_str("ppc64le"), Arch::Ppc64el);
|
||||
assert_eq!(Arch::from_str("s390x"), Arch::S390x);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_arch_alpine_name() {
|
||||
assert_eq!(Arch::Amd64.alpine_name(), "x86_64");
|
||||
assert_eq!(Arch::Arm64.alpine_name(), "aarch64");
|
||||
assert_eq!(Arch::Armhf.alpine_name(), "armv7");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_arch_oci_name() {
|
||||
assert_eq!(Arch::Amd64.oci_name(), "amd64");
|
||||
assert_eq!(Arch::Arm64.oci_name(), "arm64");
|
||||
assert_eq!(Arch::Armhf.oci_name(), "arm");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_memory_string_valid() {
|
||||
assert!(validate_memory_string("512M").is_ok());
|
||||
assert!(validate_memory_string("2G").is_ok());
|
||||
assert!(validate_memory_string("1024").is_ok());
|
||||
assert!(validate_memory_string("1T").is_ok());
|
||||
assert!(validate_memory_string("256K").is_ok());
|
||||
assert!(validate_memory_string("2g").is_ok()); // lowercase
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_memory_string_invalid() {
|
||||
assert!(validate_memory_string("").is_err());
|
||||
assert!(validate_memory_string("-1G").is_err());
|
||||
assert!(validate_memory_string("2X").is_err());
|
||||
assert!(validate_memory_string("abc").is_err());
|
||||
assert!(validate_memory_string("G").is_err());
|
||||
assert!(validate_memory_string("1.5G").is_err());
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user