mirror of
https://github.com/vhaudiquet/ecrou.git
synced 2026-11-12 22:22:44 +00:00
refactor(download): one blocking HTTP stack, no tokio runtime
The library is blocking end to end — vm::run waits on the machine for the payload's whole lifetime — so the async interior bought no concurrency (OCI layers download sequentially) and nothing any consumer shape could use; each call rented a tokio runtime with 'Failed to create Tokio runtime' as a user-visible failure mode. download.rs and registry.rs move to reqwest::blocking, folding the runtime wrappers away; distro.rs joins the same client shape, which also gives its metadata calls their first timeouts. The shared client bounds the connect and, for small metadata bodies, the whole request; transfers stay unbounded (the blocking client has no per-read window; a stalled stream is the operator's Ctrl-C). tokio and futures-util drop out as direct dependencies; tokio remains transitively under reqwest.
This commit is contained in:
1 parent
a758689bbe
commit
e4f712b782
5 files changed
+94
-174
No files matched your search
Generated
-41
@@ -362,7 +362,6 @@ dependencies = [
|
|||||||
"anyhow",
|
"anyhow",
|
||||||
"dirs",
|
"dirs",
|
||||||
"flate2",
|
"flate2",
|
||||||
"futures-util",
|
|
||||||
"libc",
|
"libc",
|
||||||
"libloading",
|
"libloading",
|
||||||
"log",
|
"log",
|
||||||
@@ -374,7 +373,6 @@ dependencies = [
|
|||||||
"tar",
|
"tar",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tokio",
|
|
||||||
"xz2",
|
"xz2",
|
||||||
"zstd",
|
"zstd",
|
||||||
]
|
]
|
||||||
@@ -502,17 +500,6 @@ version = "0.3.34"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed"
|
checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "futures-macro"
|
|
||||||
version = "0.3.34"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"syn 3.0.9",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures-sink"
|
name = "futures-sink"
|
||||||
version = "0.3.34"
|
version = "0.3.34"
|
||||||
@@ -533,7 +520,6 @@ checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-io",
|
"futures-io",
|
||||||
"futures-macro",
|
|
||||||
"futures-sink",
|
"futures-sink",
|
||||||
"futures-task",
|
"futures-task",
|
||||||
"memchr",
|
"memchr",
|
||||||
@@ -1229,14 +1215,12 @@ dependencies = [
|
|||||||
"sync_wrapper",
|
"sync_wrapper",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-rustls",
|
"tokio-rustls",
|
||||||
"tokio-util",
|
|
||||||
"tower",
|
"tower",
|
||||||
"tower-http",
|
"tower-http",
|
||||||
"tower-service",
|
"tower-service",
|
||||||
"url",
|
"url",
|
||||||
"wasm-bindgen",
|
"wasm-bindgen",
|
||||||
"wasm-bindgen-futures",
|
"wasm-bindgen-futures",
|
||||||
"wasm-streams",
|
|
||||||
"web-sys",
|
"web-sys",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -1691,21 +1675,9 @@ dependencies = [
|
|||||||
"mio",
|
"mio",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"socket2",
|
"socket2",
|
||||||
"tokio-macros",
|
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tokio-macros"
|
|
||||||
version = "2.7.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"syn 3.0.9",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokio-rustls"
|
name = "tokio-rustls"
|
||||||
version = "0.26.6"
|
version = "0.26.6"
|
||||||
@@ -1947,19 +1919,6 @@ dependencies = [
|
|||||||
"unicode-ident",
|
"unicode-ident",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "wasm-streams"
|
|
||||||
version = "0.5.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb"
|
|
||||||
dependencies = [
|
|
||||||
"futures-util",
|
|
||||||
"js-sys",
|
|
||||||
"wasm-bindgen",
|
|
||||||
"wasm-bindgen-futures",
|
|
||||||
"web-sys",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "web-sys"
|
name = "web-sys"
|
||||||
version = "0.3.106"
|
version = "0.3.106"
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ serde = { version = "1", features = ["derive"] }
|
|||||||
serde_yaml = "0.9"
|
serde_yaml = "0.9"
|
||||||
|
|
||||||
# HTTP downloads
|
# HTTP downloads
|
||||||
reqwest = { version = "0.13", features = ["blocking", "stream"] }
|
reqwest = { version = "0.13", features = ["blocking"] }
|
||||||
|
|
||||||
# Tarball extraction
|
# Tarball extraction
|
||||||
tar = "0.4"
|
tar = "0.4"
|
||||||
@@ -36,8 +36,6 @@ thiserror = "2"
|
|||||||
|
|
||||||
# Utilities
|
# Utilities
|
||||||
dirs = "6"
|
dirs = "6"
|
||||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "io-util"] }
|
|
||||||
futures-util = "0.3"
|
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
|
|
||||||
# libkrun FFI: raw ioctl for the KVM probe, runtime dlopen for the ABI
|
# libkrun FFI: raw ioctl for the KVM probe, runtime dlopen for the ABI
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use anyhow::{anyhow, Context};
|
use anyhow::{anyhow, Context};
|
||||||
|
|
||||||
@@ -198,7 +199,10 @@ pub fn checksum_artifact_url(distro: &Distro, tarball_url: &str) -> Option<Strin
|
|||||||
/// Fetch the latest Ubuntu codename from a changelogs.ubuntu.com meta-release file.
|
/// Fetch the latest Ubuntu codename from a changelogs.ubuntu.com meta-release file.
|
||||||
/// Pass the LTS-only URL to get the latest LTS, or the full URL for the latest release.
|
/// Pass the LTS-only URL to get the latest LTS, or the full URL for the latest release.
|
||||||
fn fetch_ubuntu_codename(meta_release_url: &str) -> anyhow::Result<String> {
|
fn fetch_ubuntu_codename(meta_release_url: &str) -> anyhow::Result<String> {
|
||||||
let text = reqwest::blocking::get(meta_release_url)
|
let client = crate::download::http_client(Some(Duration::from_secs(30)))?;
|
||||||
|
let text = client
|
||||||
|
.get(meta_release_url)
|
||||||
|
.send()
|
||||||
.with_context(|| format!("Failed to fetch {}", meta_release_url))?
|
.with_context(|| format!("Failed to fetch {}", meta_release_url))?
|
||||||
.text()
|
.text()
|
||||||
.with_context(|| format!("Failed to read {}", meta_release_url))?;
|
.with_context(|| format!("Failed to read {}", meta_release_url))?;
|
||||||
@@ -240,7 +244,10 @@ fn fetch_alpine_version_from_branch(branch: &str, arch: &str) -> anyhow::Result<
|
|||||||
"https://dl-cdn.alpinelinux.org/alpine/{}/releases/{}/latest-releases.yaml",
|
"https://dl-cdn.alpinelinux.org/alpine/{}/releases/{}/latest-releases.yaml",
|
||||||
branch, arch
|
branch, arch
|
||||||
);
|
);
|
||||||
let text = reqwest::blocking::get(&url)
|
let client = crate::download::http_client(Some(Duration::from_secs(30)))?;
|
||||||
|
let text = client
|
||||||
|
.get(&url)
|
||||||
|
.send()
|
||||||
.with_context(|| {
|
.with_context(|| {
|
||||||
format!(
|
format!(
|
||||||
"Failed to fetch Alpine latest-releases.yaml for branch {}",
|
"Failed to fetch Alpine latest-releases.yaml for branch {}",
|
||||||
@@ -305,7 +312,10 @@ pub fn resolve_distro_version(
|
|||||||
/// meta-release file (e.g. "24.04" → "noble", "22.04" → "jammy").
|
/// meta-release file (e.g. "24.04" → "noble", "22.04" → "jammy").
|
||||||
/// Uses the full meta-release (not -lts) so non-LTS versions are also covered.
|
/// Uses the full meta-release (not -lts) so non-LTS versions are also covered.
|
||||||
fn fetch_ubuntu_version_map() -> anyhow::Result<std::collections::HashMap<String, String>> {
|
fn fetch_ubuntu_version_map() -> anyhow::Result<std::collections::HashMap<String, String>> {
|
||||||
let text = reqwest::blocking::get("https://changelogs.ubuntu.com/meta-release")
|
let client = crate::download::http_client(Some(Duration::from_secs(30)))?;
|
||||||
|
let text = client
|
||||||
|
.get("https://changelogs.ubuntu.com/meta-release")
|
||||||
|
.send()
|
||||||
.context("Failed to fetch Ubuntu meta-release")?
|
.context("Failed to fetch Ubuntu meta-release")?
|
||||||
.text()
|
.text()
|
||||||
.context("Failed to read Ubuntu meta-release")?;
|
.context("Failed to read Ubuntu meta-release")?;
|
||||||
|
|||||||
@@ -1,17 +1,32 @@
|
|||||||
//! Download plumbing shared by both image sources: the digest hasher,
|
//! Download plumbing shared by both image sources: the digest hasher,
|
||||||
//! distro checksum files, and plain file transfers into the cache.
|
//! the shared HTTP client, distro checksum files, and plain file
|
||||||
|
//! transfers into the cache.
|
||||||
|
|
||||||
use crate::progress::{Progress, Unit};
|
use crate::progress::{Progress, Unit};
|
||||||
use anyhow::{anyhow, Context};
|
use anyhow::{anyhow, Context};
|
||||||
use futures_util::StreamExt;
|
use reqwest::blocking::Client;
|
||||||
use reqwest::Client;
|
|
||||||
use sha2::{Digest, Sha256, Sha512};
|
use sha2::{Digest, Sha256, Sha512};
|
||||||
|
use std::io::{Read, Write};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use crate::distro::{Distro, ImageSource};
|
use crate::distro::{Distro, ImageSource};
|
||||||
use crate::Result;
|
use crate::Result;
|
||||||
|
|
||||||
|
/// One HTTP client shape for the crate: a connect window, plus an
|
||||||
|
/// optional total request window. Metadata callers (checksum files,
|
||||||
|
/// token endpoints, version lookups) pass `Some` — small bodies, so a
|
||||||
|
/// total bound is safe. Transfer callers pass `None`: a slow transfer
|
||||||
|
/// of any size must finish, and the blocking client offers no per-read
|
||||||
|
/// window — a stalled stream is the operator's Ctrl-C.
|
||||||
|
pub(crate) fn http_client(request_window: Option<Duration>) -> anyhow::Result<Client> {
|
||||||
|
let mut builder = Client::builder().connect_timeout(Duration::from_secs(30));
|
||||||
|
if let Some(window) = request_window {
|
||||||
|
builder = builder.timeout(window);
|
||||||
|
}
|
||||||
|
builder.build().context("Failed to create HTTP client")
|
||||||
|
}
|
||||||
|
|
||||||
/// Path of the digest sidecar file for a cached OCI image.
|
/// Path of the digest sidecar file for a cached OCI image.
|
||||||
/// e.g. `foo.tar.gz` → `foo.tar.gz.digest`
|
/// e.g. `foo.tar.gz` → `foo.tar.gz.digest`
|
||||||
pub fn digest_sidecar(cache_path: &Path) -> PathBuf {
|
pub fn digest_sidecar(cache_path: &Path) -> PathBuf {
|
||||||
@@ -91,11 +106,6 @@ pub fn download_image(
|
|||||||
arch: &str,
|
arch: &str,
|
||||||
progress: &dyn Progress,
|
progress: &dyn Progress,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
// Create a single Tokio runtime for all async I/O in this download.
|
|
||||||
// Previously each of the two sync wrappers (download_file_sync and
|
|
||||||
// download_oci_image) created their own runtime; this consolidates them.
|
|
||||||
let rt = tokio::runtime::Runtime::new().context("Failed to create Tokio runtime")?;
|
|
||||||
|
|
||||||
match source {
|
match source {
|
||||||
ImageSource::DirectTarball { distro, version } => {
|
ImageSource::DirectTarball { distro, version } => {
|
||||||
let url = crate::distro::resolve_distro_url(distro, version.as_deref(), arch)?;
|
let url = crate::distro::resolve_distro_url(distro, version.as_deref(), arch)?;
|
||||||
@@ -107,7 +117,7 @@ pub fn download_image(
|
|||||||
// next to it. A fetch failure (missing artifact, network error)
|
// next to it. A fetch failure (missing artifact, network error)
|
||||||
// degrades to an unverified download with a warning rather than
|
// degrades to an unverified download with a warning rather than
|
||||||
// hard-failing, mirroring the digest freshness check in rootfs.
|
// hard-failing, mirroring the digest freshness check in rootfs.
|
||||||
let expected = match rt.block_on(fetch_published_checksum(distro, &url)) {
|
let expected = match fetch_published_checksum(distro, &url) {
|
||||||
Ok(expected) => Some(expected),
|
Ok(expected) => Some(expected),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
log::warn!(
|
log::warn!(
|
||||||
@@ -118,12 +128,8 @@ pub fn download_image(
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
Ok(rt.block_on(download_file_async(
|
download_file(&url, dest, progress, expected.as_deref())?;
|
||||||
&url,
|
Ok(())
|
||||||
dest,
|
|
||||||
progress,
|
|
||||||
expected.as_deref(),
|
|
||||||
))?)
|
|
||||||
}
|
}
|
||||||
ImageSource::OciImage {
|
ImageSource::OciImage {
|
||||||
registry,
|
registry,
|
||||||
@@ -132,14 +138,15 @@ pub fn download_image(
|
|||||||
architecture,
|
architecture,
|
||||||
} => {
|
} => {
|
||||||
log::debug!("Pulling OCI image: {}/{}:{}", registry, repository, tag);
|
log::debug!("Pulling OCI image: {}/{}:{}", registry, repository, tag);
|
||||||
Ok(rt.block_on(crate::registry::download_oci_image_async(
|
crate::registry::download_oci_image(
|
||||||
registry,
|
registry,
|
||||||
repository,
|
repository,
|
||||||
tag,
|
tag,
|
||||||
architecture,
|
architecture,
|
||||||
dest,
|
dest,
|
||||||
progress,
|
progress,
|
||||||
))?)
|
)?;
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
// A local archive is read in place by the prepare step; there
|
// A local archive is read in place by the prepare step; there
|
||||||
// is nothing to fetch.
|
// is nothing to fetch.
|
||||||
@@ -152,22 +159,15 @@ pub fn download_image(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Fetch the sha256 the distro publishes for the tarball at `tarball_url`.
|
/// Fetch the sha256 the distro publishes for the tarball at `tarball_url`.
|
||||||
async fn fetch_published_checksum(distro: &Distro, tarball_url: &str) -> anyhow::Result<String> {
|
fn fetch_published_checksum(distro: &Distro, tarball_url: &str) -> anyhow::Result<String> {
|
||||||
let Some(checksum_url) = crate::distro::checksum_artifact_url(distro, tarball_url) else {
|
let Some(checksum_url) = crate::distro::checksum_artifact_url(distro, tarball_url) else {
|
||||||
return Err(anyhow!("no checksum artifact for {}", tarball_url));
|
return Err(anyhow!("no checksum artifact for {}", tarball_url));
|
||||||
};
|
};
|
||||||
|
|
||||||
// Metadata call: a small body, so the read window alone bounds it.
|
let client = http_client(Some(Duration::from_secs(30)))?;
|
||||||
let client = Client::builder()
|
|
||||||
.connect_timeout(Duration::from_secs(30))
|
|
||||||
.read_timeout(Duration::from_secs(30))
|
|
||||||
.build()
|
|
||||||
.context("Failed to create HTTP client")?;
|
|
||||||
|
|
||||||
let response = client
|
let response = client
|
||||||
.get(&checksum_url)
|
.get(&checksum_url)
|
||||||
.send()
|
.send()
|
||||||
.await
|
|
||||||
.with_context(|| format!("Failed to fetch checksum artifact {}", checksum_url))?;
|
.with_context(|| format!("Failed to fetch checksum artifact {}", checksum_url))?;
|
||||||
|
|
||||||
if !response.status().is_success() {
|
if !response.status().is_success() {
|
||||||
@@ -178,7 +178,7 @@ async fn fetch_published_checksum(distro: &Distro, tarball_url: &str) -> anyhow:
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
let text = response.text().await?;
|
let text = response.text()?;
|
||||||
let filename = tarball_url.rsplit('/').next().unwrap_or("");
|
let filename = tarball_url.rsplit('/').next().unwrap_or("");
|
||||||
expected_digest_from_checksums(&text, filename).ok_or_else(|| {
|
expected_digest_from_checksums(&text, filename).ok_or_else(|| {
|
||||||
anyhow!(
|
anyhow!(
|
||||||
@@ -204,7 +204,7 @@ fn expected_digest_from_checksums(text: &str, filename: &str) -> Option<String>
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn download_file_async(
|
fn download_file(
|
||||||
url: &str,
|
url: &str,
|
||||||
dest: &Path,
|
dest: &Path,
|
||||||
progress: &dyn Progress,
|
progress: &dyn Progress,
|
||||||
@@ -212,20 +212,8 @@ async fn download_file_async(
|
|||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
log::debug!("Downloading: {}", url);
|
log::debug!("Downloading: {}", url);
|
||||||
|
|
||||||
// No total timeout: a slow transfer of any size must finish. The
|
let client = http_client(None)?;
|
||||||
// connect window and the per-read window catch dead peers and
|
let mut response = client.get(url).send().context("Failed to start download")?;
|
||||||
// stalled streams instead.
|
|
||||||
let client = Client::builder()
|
|
||||||
.connect_timeout(Duration::from_secs(30))
|
|
||||||
.read_timeout(Duration::from_secs(60))
|
|
||||||
.build()
|
|
||||||
.context("Failed to create HTTP client")?;
|
|
||||||
|
|
||||||
let response = client
|
|
||||||
.get(url)
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.context("Failed to start download")?;
|
|
||||||
|
|
||||||
if !response.status().is_success() {
|
if !response.status().is_success() {
|
||||||
return Err(anyhow!("Download failed: HTTP {}", response.status()));
|
return Err(anyhow!("Download failed: HTTP {}", response.status()));
|
||||||
@@ -235,9 +223,7 @@ async fn download_file_async(
|
|||||||
|
|
||||||
// Create parent directories
|
// Create parent directories
|
||||||
if let Some(parent) = dest.parent() {
|
if let Some(parent) = dest.parent() {
|
||||||
tokio::fs::create_dir_all(parent)
|
std::fs::create_dir_all(parent).context("Failed to create cache directory")?;
|
||||||
.await
|
|
||||||
.context("Failed to create cache directory")?;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
progress.begin(
|
progress.begin(
|
||||||
@@ -256,17 +242,19 @@ async fn download_file_async(
|
|||||||
// reaches the cache; on any failure after the temp file is created we
|
// reaches the cache; on any failure after the temp file is created we
|
||||||
// remove it so stale partial files don't accumulate.
|
// remove it so stale partial files don't accumulate.
|
||||||
let mut hasher = DownloadHasher::new();
|
let mut hasher = DownloadHasher::new();
|
||||||
let result: anyhow::Result<()> = async {
|
let result: anyhow::Result<()> = (|| {
|
||||||
let mut file = tokio::fs::File::create(&temp_dest)
|
let mut file =
|
||||||
.await
|
std::fs::File::create(&temp_dest).context("Failed to create temporary file")?;
|
||||||
.context("Failed to create temporary file")?;
|
|
||||||
|
|
||||||
let mut stream = response.bytes_stream();
|
let mut buf = [0u8; 64 * 1024];
|
||||||
while let Some(chunk) = stream.next().await {
|
loop {
|
||||||
let chunk = chunk.context("Download interrupted")?;
|
let n = response.read(&mut buf).context("Download interrupted")?;
|
||||||
tokio::io::copy(&mut &chunk[..], &mut file).await?;
|
if n == 0 {
|
||||||
hasher.update(&chunk);
|
break;
|
||||||
progress.advance(chunk.len() as u64);
|
}
|
||||||
|
file.write_all(&buf[..n]).context("Failed to write chunk")?;
|
||||||
|
hasher.update(&buf[..n]);
|
||||||
|
progress.advance(n as u64);
|
||||||
}
|
}
|
||||||
|
|
||||||
progress.finish();
|
progress.finish();
|
||||||
@@ -275,18 +263,16 @@ async fn download_file_async(
|
|||||||
log::debug!("Verified content digest: {}", dest.display());
|
log::debug!("Verified content digest: {}", dest.display());
|
||||||
}
|
}
|
||||||
|
|
||||||
tokio::fs::rename(&temp_dest, dest)
|
std::fs::rename(&temp_dest, dest)
|
||||||
.await
|
|
||||||
.context("Failed to move partial download to final destination")?;
|
.context("Failed to move partial download to final destination")?;
|
||||||
log::debug!("Download complete: {}", dest.display());
|
log::debug!("Download complete: {}", dest.display());
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
})();
|
||||||
.await;
|
|
||||||
|
|
||||||
if result.is_err() {
|
if result.is_err() {
|
||||||
// Best-effort cleanup; ignore errors (file may not exist if creation failed).
|
// Best-effort cleanup; ignore errors (file may not exist if creation failed).
|
||||||
let _ = tokio::fs::remove_file(&temp_dest).await;
|
let _ = std::fs::remove_file(&temp_dest);
|
||||||
}
|
}
|
||||||
|
|
||||||
result
|
result
|
||||||
|
|||||||
@@ -3,12 +3,13 @@
|
|||||||
//! and content-addressed layer blobs bundled into a cache entry.
|
//! and content-addressed layer blobs bundled into a cache entry.
|
||||||
|
|
||||||
use anyhow::{anyhow, Context};
|
use anyhow::{anyhow, Context};
|
||||||
use futures_util::StreamExt;
|
use reqwest::blocking::Client;
|
||||||
use reqwest::Client;
|
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
use std::io::{Read, Write};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use crate::download::http_client;
|
||||||
use crate::download::{digest_sidecar, DownloadHasher};
|
use crate::download::{digest_sidecar, DownloadHasher};
|
||||||
use crate::progress::{Progress, Unit};
|
use crate::progress::{Progress, Unit};
|
||||||
use crate::{Error, Result};
|
use crate::{Error, Result};
|
||||||
@@ -17,25 +18,13 @@ use crate::{Error, Result};
|
|||||||
/// layers. The registry returns the content-addressable digest in the
|
/// layers. The registry returns the content-addressable digest in the
|
||||||
/// `Docker-Content-Digest` response header of any manifest GET.
|
/// `Docker-Content-Digest` response header of any manifest GET.
|
||||||
pub(crate) fn fetch_oci_digest(registry: &str, repository: &str, tag: &str) -> Result<String> {
|
pub(crate) fn fetch_oci_digest(registry: &str, repository: &str, tag: &str) -> Result<String> {
|
||||||
let digest = tokio::runtime::Runtime::new()
|
fetch_oci_digest_inner(registry, repository, tag).map_err(Error::Other)
|
||||||
.context("Failed to create Tokio runtime")?
|
|
||||||
.block_on(fetch_oci_digest_async(registry, repository, tag))?;
|
|
||||||
Ok(digest)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn fetch_oci_digest_async(
|
fn fetch_oci_digest_inner(registry: &str, repository: &str, tag: &str) -> anyhow::Result<String> {
|
||||||
registry: &str,
|
// Metadata: a small body, so a total request bound is safe.
|
||||||
repository: &str,
|
let client = http_client(Some(Duration::from_secs(30)))?;
|
||||||
tag: &str,
|
let token = get_auth_token(&client, registry, repository)?;
|
||||||
) -> anyhow::Result<String> {
|
|
||||||
// Metadata call: a small body, so the read window alone bounds it.
|
|
||||||
let client = Client::builder()
|
|
||||||
.connect_timeout(Duration::from_secs(30))
|
|
||||||
.read_timeout(Duration::from_secs(30))
|
|
||||||
.build()
|
|
||||||
.context("Failed to create HTTP client")?;
|
|
||||||
|
|
||||||
let token = get_auth_token(&client, registry, repository).await?;
|
|
||||||
|
|
||||||
let manifest_url = if registry == "docker.io" {
|
let manifest_url = if registry == "docker.io" {
|
||||||
format!(
|
format!(
|
||||||
@@ -57,7 +46,6 @@ async fn fetch_oci_digest_async(
|
|||||||
)
|
)
|
||||||
.header("Authorization", format!("Bearer {}", token))
|
.header("Authorization", format!("Bearer {}", token))
|
||||||
.send()
|
.send()
|
||||||
.await
|
|
||||||
.context("Failed to fetch manifest for digest check")?;
|
.context("Failed to fetch manifest for digest check")?;
|
||||||
|
|
||||||
if !response.status().is_success() {
|
if !response.status().is_success() {
|
||||||
@@ -75,7 +63,7 @@ async fn fetch_oci_digest_async(
|
|||||||
.ok_or_else(|| anyhow!("Registry did not return a Docker-Content-Digest header"))
|
.ok_or_else(|| anyhow!("Registry did not return a Docker-Content-Digest header"))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn download_oci_image_async(
|
pub(crate) fn download_oci_image(
|
||||||
registry: &str,
|
registry: &str,
|
||||||
repository: &str,
|
repository: &str,
|
||||||
tag: &str,
|
tag: &str,
|
||||||
@@ -83,17 +71,8 @@ pub(crate) async fn download_oci_image_async(
|
|||||||
dest: &Path,
|
dest: &Path,
|
||||||
progress: &dyn Progress,
|
progress: &dyn Progress,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
// No total timeout: a slow transfer of any size must finish. The
|
let client = http_client(None)?;
|
||||||
// connect window and the per-read window catch dead peers and
|
let digest = try_download_oci_image(&client, registry, repository, tag, arch, dest, progress)?;
|
||||||
// stalled streams instead.
|
|
||||||
let client = Client::builder()
|
|
||||||
.connect_timeout(Duration::from_secs(30))
|
|
||||||
.read_timeout(Duration::from_secs(60))
|
|
||||||
.build()
|
|
||||||
.context("Failed to create HTTP client")?;
|
|
||||||
|
|
||||||
let digest =
|
|
||||||
try_download_oci_image(&client, registry, repository, tag, arch, dest, progress).await?;
|
|
||||||
|
|
||||||
// Persist the manifest digest so the next run can skip the download when
|
// Persist the manifest digest so the next run can skip the download when
|
||||||
// the tag hasn't changed (see fetch_oci_digest / digest_sidecar).
|
// the tag hasn't changed (see fetch_oci_digest / digest_sidecar).
|
||||||
@@ -104,7 +83,7 @@ pub(crate) async fn download_oci_image_async(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn try_download_oci_image(
|
fn try_download_oci_image(
|
||||||
client: &Client,
|
client: &Client,
|
||||||
registry: &str,
|
registry: &str,
|
||||||
repository: &str,
|
repository: &str,
|
||||||
@@ -114,7 +93,7 @@ async fn try_download_oci_image(
|
|||||||
progress: &dyn Progress,
|
progress: &dyn Progress,
|
||||||
) -> anyhow::Result<String> {
|
) -> anyhow::Result<String> {
|
||||||
// Get authentication token for the registry
|
// Get authentication token for the registry
|
||||||
let token = get_auth_token(client, registry, repository).await?;
|
let token = get_auth_token(client, registry, repository)?;
|
||||||
|
|
||||||
// Construct manifest URL based on registry
|
// Construct manifest URL based on registry
|
||||||
let manifest_url = if registry == "docker.io" {
|
let manifest_url = if registry == "docker.io" {
|
||||||
@@ -135,7 +114,6 @@ async fn try_download_oci_image(
|
|||||||
.header("Accept", "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json")
|
.header("Accept", "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json")
|
||||||
.header("Authorization", format!("Bearer {}", token))
|
.header("Authorization", format!("Bearer {}", token))
|
||||||
.send()
|
.send()
|
||||||
.await
|
|
||||||
.context("Failed to get image manifest")?;
|
.context("Failed to get image manifest")?;
|
||||||
|
|
||||||
if !response.status().is_success() {
|
if !response.status().is_success() {
|
||||||
@@ -156,7 +134,7 @@ async fn try_download_oci_image(
|
|||||||
.map(|s| s.to_string())
|
.map(|s| s.to_string())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
let body = response.text().await?;
|
let body = response.text()?;
|
||||||
let manifest: Value = serde_json::from_str(&body).context("Failed to parse manifest JSON")?;
|
let manifest: Value = serde_json::from_str(&body).context("Failed to parse manifest JSON")?;
|
||||||
|
|
||||||
// Check if this is a manifest list (multi-arch)
|
// Check if this is a manifest list (multi-arch)
|
||||||
@@ -208,7 +186,6 @@ async fn try_download_oci_image(
|
|||||||
.header("Accept", "application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json")
|
.header("Accept", "application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json")
|
||||||
.header("Authorization", format!("Bearer {}", token))
|
.header("Authorization", format!("Bearer {}", token))
|
||||||
.send()
|
.send()
|
||||||
.await
|
|
||||||
.context("Failed to get architecture manifest")?;
|
.context("Failed to get architecture manifest")?;
|
||||||
|
|
||||||
if !arch_response.status().is_success() {
|
if !arch_response.status().is_success() {
|
||||||
@@ -218,7 +195,7 @@ async fn try_download_oci_image(
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
let arch_body = arch_response.text().await?;
|
let arch_body = arch_response.text()?;
|
||||||
|
|
||||||
// The arch manifest was requested by digest: verify the response
|
// The arch manifest was requested by digest: verify the response
|
||||||
// matches, since the layer digests read from it are trusted next.
|
// matches, since the layer digests read from it are trusted next.
|
||||||
@@ -293,8 +270,7 @@ async fn try_download_oci_image(
|
|||||||
&label,
|
&label,
|
||||||
layer_digest,
|
layer_digest,
|
||||||
progress,
|
progress,
|
||||||
)
|
)?;
|
||||||
.await?;
|
|
||||||
layer_names.push(layer_name);
|
layer_names.push(layer_name);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -348,11 +324,7 @@ async fn try_download_oci_image(
|
|||||||
Ok(tag_digest)
|
Ok(tag_digest)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_auth_token(
|
fn get_auth_token(client: &Client, registry: &str, repository: &str) -> anyhow::Result<String> {
|
||||||
client: &Client,
|
|
||||||
registry: &str,
|
|
||||||
repository: &str,
|
|
||||||
) -> anyhow::Result<String> {
|
|
||||||
// Docker Hub has a well-known, stable auth endpoint.
|
// Docker Hub has a well-known, stable auth endpoint.
|
||||||
if registry == "docker.io" {
|
if registry == "docker.io" {
|
||||||
let url = format!(
|
let url = format!(
|
||||||
@@ -360,7 +332,7 @@ async fn get_auth_token(
|
|||||||
repository
|
repository
|
||||||
);
|
);
|
||||||
log::debug!("Getting auth token from: {}", url);
|
log::debug!("Getting auth token from: {}", url);
|
||||||
return fetch_bearer_token(client, &url).await;
|
return fetch_bearer_token(client, &url);
|
||||||
}
|
}
|
||||||
|
|
||||||
// For every other registry follow the OCI Distribution Spec §4.2:
|
// For every other registry follow the OCI Distribution Spec §4.2:
|
||||||
@@ -371,7 +343,6 @@ async fn get_auth_token(
|
|||||||
let probe = client
|
let probe = client
|
||||||
.get(&probe_url)
|
.get(&probe_url)
|
||||||
.send()
|
.send()
|
||||||
.await
|
|
||||||
.with_context(|| format!("Failed to probe registry at {}", probe_url))?;
|
.with_context(|| format!("Failed to probe registry at {}", probe_url))?;
|
||||||
|
|
||||||
match probe.status().as_u16() {
|
match probe.status().as_u16() {
|
||||||
@@ -412,7 +383,7 @@ async fn get_auth_token(
|
|||||||
realm, service, repository
|
realm, service, repository
|
||||||
);
|
);
|
||||||
log::debug!("Getting auth token from: {}", token_url);
|
log::debug!("Getting auth token from: {}", token_url);
|
||||||
fetch_bearer_token(client, &token_url).await
|
fetch_bearer_token(client, &token_url)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parse a `Bearer realm="...",service="..."[,scope="..."]` header value.
|
/// Parse a `Bearer realm="...",service="..."[,scope="..."]` header value.
|
||||||
@@ -452,11 +423,10 @@ fn parse_www_authenticate(header: &str) -> Option<(String, String)> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Fetch a bearer token from a fully-constructed token URL.
|
/// Fetch a bearer token from a fully-constructed token URL.
|
||||||
async fn fetch_bearer_token(client: &Client, token_url: &str) -> anyhow::Result<String> {
|
fn fetch_bearer_token(client: &Client, token_url: &str) -> anyhow::Result<String> {
|
||||||
let response = client
|
let response = client
|
||||||
.get(token_url)
|
.get(token_url)
|
||||||
.send()
|
.send()
|
||||||
.await
|
|
||||||
.context("Failed to request auth token")?;
|
.context("Failed to request auth token")?;
|
||||||
|
|
||||||
if !response.status().is_success() {
|
if !response.status().is_success() {
|
||||||
@@ -464,7 +434,7 @@ async fn fetch_bearer_token(client: &Client, token_url: &str) -> anyhow::Result<
|
|||||||
return Ok(String::new());
|
return Ok(String::new());
|
||||||
}
|
}
|
||||||
|
|
||||||
let body = response.text().await?;
|
let body = response.text()?;
|
||||||
let json: Value = serde_json::from_str(&body).context("Failed to parse auth token response")?;
|
let json: Value = serde_json::from_str(&body).context("Failed to parse auth token response")?;
|
||||||
|
|
||||||
// OCI spec uses "token"; Docker also emits "access_token".
|
// OCI spec uses "token"; Docker also emits "access_token".
|
||||||
@@ -475,7 +445,7 @@ async fn fetch_bearer_token(client: &Client, token_url: &str) -> anyhow::Result<
|
|||||||
.ok_or_else(|| anyhow!("No token field in auth response: {}", body))
|
.ok_or_else(|| anyhow!("No token field in auth response: {}", body))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn download_blob_with_auth(
|
fn download_blob_with_auth(
|
||||||
client: &Client,
|
client: &Client,
|
||||||
url: &str,
|
url: &str,
|
||||||
dest: &Path,
|
dest: &Path,
|
||||||
@@ -489,7 +459,7 @@ async fn download_blob_with_auth(
|
|||||||
request = request.header("Authorization", format!("Bearer {}", token));
|
request = request.header("Authorization", format!("Bearer {}", token));
|
||||||
}
|
}
|
||||||
|
|
||||||
let response = request.send().await.context("Failed to start download")?;
|
let mut response = request.send().context("Failed to start download")?;
|
||||||
|
|
||||||
if !response.status().is_success() {
|
if !response.status().is_success() {
|
||||||
return Err(anyhow!(
|
return Err(anyhow!(
|
||||||
@@ -502,24 +472,21 @@ async fn download_blob_with_auth(
|
|||||||
|
|
||||||
progress.begin(label, Unit::Bytes, (total_size > 0).then_some(total_size));
|
progress.begin(label, Unit::Bytes, (total_size > 0).then_some(total_size));
|
||||||
|
|
||||||
let mut file = tokio::fs::File::create(dest)
|
let mut file = std::fs::File::create(dest).context("Failed to create destination file")?;
|
||||||
.await
|
|
||||||
.context("Failed to create destination file")?;
|
|
||||||
|
|
||||||
let mut hasher = DownloadHasher::new();
|
let mut hasher = DownloadHasher::new();
|
||||||
let mut stream = response.bytes_stream();
|
let mut buf = [0u8; 64 * 1024];
|
||||||
use tokio::io::AsyncWriteExt;
|
loop {
|
||||||
|
let n = response.read(&mut buf).context("Failed to read chunk")?;
|
||||||
while let Some(chunk) = stream.next().await {
|
if n == 0 {
|
||||||
let chunk = chunk.context("Failed to read chunk")?;
|
break;
|
||||||
file.write_all(&chunk)
|
}
|
||||||
.await
|
file.write_all(&buf[..n]).context("Failed to write chunk")?;
|
||||||
.context("Failed to write chunk")?;
|
hasher.update(&buf[..n]);
|
||||||
hasher.update(&chunk);
|
progress.advance(n as u64);
|
||||||
progress.advance(chunk.len() as u64);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
file.flush().await?;
|
file.flush().context("Failed to flush the layer file")?;
|
||||||
progress.finish();
|
progress.finish();
|
||||||
|
|
||||||
// The blob was addressed by this digest in the manifest; a mismatch means
|
// The blob was addressed by this digest in the manifest; a mismatch means
|
||||||
|
|||||||
Reference in new issue
Block a user