From 0e1e37b2e6ba1ff21ee4b5ec8f33d2357aa9f801 Mon Sep 17 00:00:00 2001 From: Valentin Haudiquet Date: Tue, 29 Sep 2026 14:22:49 +0200 Subject: [PATCH] deb: share the apt download cache across builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the chroot now per-run scratch, every build re-downloaded the packages its dependency install needed — the very cost the chroot reuse used to hide. Every chroot's /var/cache/apt/archives is now a bind mount over a shared cache at /var/tmp/pkh/apt-cache/-/ (the chroot's architecture, like the tarball cache), which one run's installs fill and the next runs reuse: apt keeps downloaded packages through a dropped conf (APT::Keep-Downloaded-Packages), and an apt-get autoclean at the end of each run drops the debs the archive no longer serves. pkh prune removes caches untouched for a week. The run holds a host-side flock on the cache for its whole lifetime: apt's own archives lock is acquired inside the chroot's pid namespace, where a concurrent build's apt shows the holder as "process 0" — an invisible process it refuses to wait for — so pkh serializes the traffic itself; concurrent builds of the same target wait for each other. Any bind-mount failure falls back to the chroot's own cache with a warning (correct, just slower), and the temp-chroot teardown now unmounts everything under the chroot before rm -rf, so the removal can never reach through the cache bind mount and delete the shared cache itself. --- README.md | 8 ++ src/deb/ephemeral.rs | 276 ++++++++++++++++++++++++++++++++++++------- src/deb/local.rs | 17 +++ src/prune.rs | 109 +++++++++++++++-- 4 files changed, 363 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 5e293af..bcc46a3 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,14 @@ so records stay small. What `--incremental` buys is the record's **staged tree** (`tree-upper/`): the overlay upperdir where the build's writes land, holding the compiled objects. +Fresh chroots do not mean fresh downloads: every chroot's +`/var/cache/apt/archives` is a bind mount over a shared apt download +cache at `/var/tmp/pkh/apt-cache/-/`, which the installs +of one run fill and the next runs reuse (apt is configured to keep the +downloaded packages, and an `apt-get autoclean` at the end of each run +drops the debs the archive no longer serves). `pkh prune` removes the +caches untouched for a week. + `--incremental` forks a previous record — a copy, the base is never modified — so a failed run can never destroy the last-good state it was built from. The staged tree is carried only when the base's recorded diff --git a/src/deb/ephemeral.rs b/src/deb/ephemeral.rs index ecbea69..d798ad7 100644 --- a/src/deb/ephemeral.rs +++ b/src/deb/ephemeral.rs @@ -6,6 +6,7 @@ use crate::report::BuildView; use directories::ProjectDirs; use std::error::Error; use std::fs; +use std::os::unix::io::AsRawFd; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::Arc; @@ -238,8 +239,8 @@ fn is_mountpoint(path: &Path) -> bool { } /// The cached chroot tarball path for a series/arch, as used by the -/// bootstrap: the resume path compares it against the tarball recorded in -/// the session manifest. +/// bootstrap: the record's fork gate compares the tarball's digest +/// against the sha recorded in a base's manifest. pub(crate) fn chroot_tarball_path( series: &str, arch: Option<&str>, @@ -254,6 +255,40 @@ pub(crate) fn chroot_tarball_path( Ok(proj_dirs.cache_dir().join(tarball_filename)) } +/// Root of the shared apt download caches (`/var/tmp/pkh/apt-cache`), +/// one `-` subdirectory per target. +pub(crate) const APT_CACHE_ROOT: &str = "/var/tmp/pkh/apt-cache"; + +/// The shared apt download cache of a `series`/`arch` target: +/// `/var/tmp/pkh/apt-cache/-/`, bind-mounted over each +/// chroot's `/var/cache/apt/archives` so dependency installs reuse the +/// `.debs` previous runs downloaded instead of fetching them again. +/// `arch` is the chroot's architecture (the foreign one for qemu-binfmt +/// builds, the host one otherwise), keyed like the chroot tarball cache. +pub(crate) fn apt_cache_dir(series: &str, arch: Option<&str>) -> PathBuf { + // The host architecture when the chroot is not a foreign one. + let arch = arch + .map(str::to_string) + .unwrap_or_else(crate::get_current_arch); + PathBuf::from(APT_CACHE_ROOT).join(format!("{series}-{arch}")) +} + +/// Name of the apt configuration file that keeps downloaded packages +/// (without it apt deletes the `.debs` it installs, and the shared +/// cache would never fill). +pub(crate) const APT_KEEP_CONF: &str = "99pkh-shared-download-cache"; + +/// Write the apt configuration keeping downloaded packages into the +/// chroot, so installs populate the (shared) download cache. +pub(crate) fn write_apt_keep_conf(chroot_path: &Path) -> std::io::Result<()> { + let conf_dir = chroot_path.join("etc/apt/apt.conf.d"); + fs::create_dir_all(&conf_dir)?; + fs::write( + conf_dir.join(APT_KEEP_CONF), + "APT::Keep-Downloaded-Packages \"true\";\n", + ) +} + /// Unmount `path`, falling back to a lazy unmount if the first attempt fails /// because something still holds the mount busy (e.g. an interrupted child /// that has not exited yet); returns whether the mount is gone @@ -349,6 +384,12 @@ pub struct EphemeralContextGuard { /// Whether the record's staged tree is retained after the build /// (`pkh deb --incremental`). keep_upper: bool, + /// The host-side lock of the shared apt download cache, held while + /// its bind mount is up (see + /// [`EphemeralContextGuard::share_apt_cache`]); released when the + /// guard drops, together with the mount. Held for its drop side + /// effect alone, hence the underscore: it is never read. + _apt_cache_lock: Option, } impl EphemeralContextGuard { @@ -456,25 +497,34 @@ impl EphemeralContextGuard { }; // Download and extract the chroot tarball — always fresh; the - // record's chroot directory is per-run scratch. - if let Err(e) = - Self::download_and_extract_chroot(series, arch, &chroot_path, base_ctx.clone(), view) - .await + // record's chroot directory is per-run scratch. The shared apt + // cache's lock (when the cache is shared) is held from here on. + let apt_cache_lock = match Self::download_and_extract_chroot( + series, + arch, + &chroot_path, + base_ctx.clone(), + view, + ) + .await { - // On a Ctrl+C the interrupt watchdog owns the tree: keep the - // hook registered (forgetting the guard) so it removes the - // partial directory, instead of the historical behavior of - // leaving it in place. Without an interrupt this is a plain - // bootstrap failure and the partial directory stays, as before. - if crate::interrupt::interrupted() - && let Some(hook) = cleanup_hook - { - std::mem::forget(hook); - } else { - drop(cleanup_hook); + Ok(lock) => lock, + Err(e) => { + // On a Ctrl+C the interrupt watchdog owns the tree: keep the + // hook registered (forgetting the guard) so it removes the + // partial directory, instead of the historical behavior of + // leaving it in place. Without an interrupt this is a plain + // bootstrap failure and the partial directory stays, as before. + if crate::interrupt::interrupted() + && let Some(hook) = cleanup_hook + { + std::mem::forget(hook); + } else { + drop(cleanup_hook); + } + return Err(e); } - return Err(e); - } + }; // Record the tarball digest this environment came from: the fork // gate of a later `--incremental` run compares it against the @@ -514,6 +564,7 @@ impl EphemeralContextGuard { cleanup_hook, session, keep_upper, + _apt_cache_lock: apt_cache_lock, }) } @@ -533,7 +584,7 @@ impl EphemeralContextGuard { chroot_path: &PathBuf, ctx: Arc, view: &dyn BuildView, - ) -> Result<(), Box> { + ) -> Result, Box> { // The chroot is per-run scratch: nothing is reused, so never // extract over stale content. clear_dir(chroot_path)?; @@ -612,21 +663,26 @@ impl EphemeralContextGuard { enter_phase(view, Phase::ExtractingChroot); Self::extract_tarball(&tarball_path, chroot_path, view)?; - // Create device nodes in the chroot and bind-mount /proc - Self::finalize_chroot(chroot_path, ctx_for_devices, view)?; + // Create device nodes, bind-mount /proc and share the apt cache + let apt_cache_lock = + Self::finalize_chroot(series, arch, chroot_path, ctx_for_devices, view)?; // The bootstrap's tarball digest is recorded by the caller (the // record's fork gate compares it against the cache later). - Ok(()) + Ok(apt_cache_lock) } - /// Device nodes and the /proc bind mount for a fresh chroot: - /// idempotent finalization, safe to run on every run. + /// Device nodes, the /proc bind mount and the shared apt download + /// cache for a fresh chroot: idempotent finalization, safe to run on + /// every run. Returns the shared cache's lock, held while the mount + /// is up (see [`EphemeralContextGuard::share_apt_cache`]). fn finalize_chroot( + series: &str, + arch: Option<&str>, chroot_path: &Path, ctx: Arc, view: &dyn BuildView, - ) -> Result<(), Box> { + ) -> Result, Box> { if !chroot_path.join("dev/null").exists() { // Create device nodes in the chroot log::debug!("Creating device nodes in chroot..."); @@ -642,9 +698,112 @@ impl EphemeralContextGuard { log::debug!("/proc is already bind-mounted in the chroot"); } else { log::debug!("Bind-mounting /proc into chroot..."); - Self::bind_mount_proc(chroot_path, ctx)?; + Self::bind_mount_proc(chroot_path, ctx.clone())?; + } + + Ok(Self::share_apt_cache(series, arch, chroot_path, ctx)) + } + + /// Bind-mount the shared apt download cache of `series`/`arch` over + /// the chroot's `/var/cache/apt/archives`, so dependency installs + /// reuse the `.debs` previous runs downloaded instead of fetching + /// them again. The apt conf keeping downloaded packages + /// ([`write_apt_keep_conf`]) makes the installs populate it. + /// + /// While the mount is up, the run holds a host-side flock on the + /// cache's lock file (returned to the caller, released when the + /// guard drops): apt's own `archives/lock` is acquired inside the + /// chroot's pid namespace, where a concurrent build's apt appears + /// as "held by process 0" — an invisible process it refuses to wait + /// for — so pkh serializes the apt traffic of builds sharing one + /// cache itself. Concurrent builds of the same target wait for each + /// other instead of racing apt. + /// + /// Any failure is a warning, never an error: the chroot keeps its + /// own cache dir and everything stays correct, just slower. + fn share_apt_cache( + series: &str, + arch: Option<&str>, + chroot_path: &Path, + ctx: Arc, + ) -> Option { + let cache = apt_cache_dir(series, arch); + if let Err(e) = fs::create_dir_all(cache.join("partial")) { + log::warn!( + "Cannot create the shared apt cache {}: {e}; using the \ + chroot's own cache", + cache.display() + ); + return None; + } + let archives = chroot_path.join("var/cache/apt/archives"); + if let Err(e) = + fs::create_dir_all(&archives).and_then(|()| write_apt_keep_conf(chroot_path)) + { + log::warn!( + "Cannot prepare the chroot's apt cache for sharing: {e}; \ + using the chroot's own cache" + ); + return None; + } + + // Serialize with the other runs sharing this cache BEFORE apt + // can run in the chroot. A held lock is a live run: wait for it + // (a crashed pkh releases its flock with its process). + let lock_path = cache.with_extension("lock"); + let lock = match fs::OpenOptions::new() + .create(true) + .append(true) + .open(&lock_path) + { + Ok(file) => file, + Err(e) => { + log::warn!( + "Cannot open the shared apt cache lock {}: {e}; using \ + the chroot's own cache", + lock_path.display() + ); + return None; + } + }; + let taken = unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX) }; + if taken != 0 { + log::warn!( + "Cannot lock the shared apt cache {}: {}; using the \ + chroot's own cache", + lock_path.display(), + std::io::Error::last_os_error() + ); + return None; + } + + // Check if we're running as root to avoid unnecessary sudo + let is_root = crate::utils::root::is_root().unwrap_or(false); + let mut cmd = ctx.command(if is_root { "mount" } else { "sudo" }); + if !is_root { + cmd.arg("mount"); + } + let status = cmd.arg("--bind").arg(&cache).arg(&archives).status(); + match status { + Ok(status) if status.success() => { + log::debug!( + "Bind-mounted the shared apt cache {} over {}", + cache.display(), + archives.display() + ); + Some(lock) + } + outcome => { + log::warn!( + "Cannot bind-mount the shared apt cache {} over {}: \ + {:?}; using the chroot's own cache", + cache.display(), + archives.display(), + outcome + ); + None // dropping the lock: nothing is shared + } } - Ok(()) } async fn download_chroot_tarball( @@ -952,21 +1111,22 @@ impl Drop for EphemeralContextGuard { "Build succeeded, removing chroot directory: {}", self.chroot_path.display() ); - // Check if we're running as root to avoid unnecessary sudo let is_root = crate::utils::root::is_root().unwrap_or(false); - // Unmount /proc from chroot before removing (ignore errors) - let proc_path = self.chroot_path.join("proc"); - let _ = if is_root { - self.base_ctx.command("umount").arg(&proc_path).status() - } else { - self.base_ctx - .command("sudo") - .arg("umount") - .arg(&proc_path) - .status() - }; + // Unmount everything under the chroot before removing it: + // /proc, and the shared apt download cache bind-mounted over + // /var/cache/apt/archives — an rm -rf through that mount + // would delete the shared cache itself. + for mount in host_mounts_under(&self.chroot_path).into_iter().rev() { + if !unmount_path(&mount, is_root) { + log::warn!( + "Failed to unmount {} under {}", + mount.display(), + self.chroot_path.display() + ); + } + } let result = if is_root { self.base_ctx @@ -1156,4 +1316,40 @@ mod chroot_cleanup_tests { let read = std::fs::read_dir(&cleared).unwrap(); assert_eq!(read.count(), 0, "the directory must be empty"); } + + /// The shared apt cache is keyed per series and chroot architecture + /// (the foreign arch for qemu-binfmt builds, the host one + /// otherwise), like the chroot tarball cache. + #[test] + fn apt_cache_dirs_are_keyed_per_series_and_arch() { + assert_eq!( + apt_cache_dir("noble", Some("riscv64")), + PathBuf::from("/var/tmp/pkh/apt-cache/noble-riscv64") + ); + let native = apt_cache_dir("noble", None); + assert_eq!( + native, + PathBuf::from(format!( + "/var/tmp/pkh/apt-cache/noble-{}", + crate::get_current_arch() + )) + ); + assert_ne!(native, apt_cache_dir("sid", None), "per series"); + } + + /// The apt configuration keeping downloaded packages lands in the + /// chroot's apt.conf.d, so installs populate the shared cache. + #[test] + fn apt_keep_conf_lands_in_the_chroot() { + let dir = tempfile::tempdir().unwrap(); + let chroot = dir.path().join("chroot"); + write_apt_keep_conf(&chroot).unwrap(); + let conf = chroot.join("etc/apt/apt.conf.d").join(APT_KEEP_CONF); + let content = fs::read_to_string(&conf).unwrap(); + assert_eq!(content, "APT::Keep-Downloaded-Packages \"true\";\n"); + + // Idempotent: a second write does not fail or duplicate. + write_apt_keep_conf(&chroot).unwrap(); + assert_eq!(fs::read_to_string(&conf).unwrap(), content); + } } diff --git a/src/deb/local.rs b/src/deb/local.rs index 636647e..e925d6a 100644 --- a/src/deb/local.rs +++ b/src/deb/local.rs @@ -410,6 +410,23 @@ pub async fn build( s.record_phase("binary", None); } + // Trim the shared download cache before the chroot goes away: drop + // the .debs apt can no longer download (removed from the archive), + // keep the rest for the next run of this target. Best-effort — a + // failure costs cache space, never the build. + let status = cap( + ctx.command("apt-get") + .envs(env.clone()) + .arg("-y") + .arg("autoclean"), + &sink, + ) + .status(); + match status { + Ok(status) if status.success() => {} + outcome => log::warn!("apt-get autoclean failed ({outcome:?})"), + } + Ok(artifacts) } diff --git a/src/prune.rs b/src/prune.rs index 58b542a..438051d 100644 --- a/src/prune.rs +++ b/src/prune.rs @@ -10,11 +10,17 @@ //! bind-mounted `/proc` and overlay filesystems, so they require careful //! unmounting before they can be removed. //! - **Build records** under `/var/tmp/pkh/sessions///` -//! (see [`crate::deb::session`]): one immutable record per build; the +//! (see [`crate::deb::session`]): one record per build (metadata and +//! log for every run, the staged tree of `--incremental` ones); the //! default policy removes the ones untouched for longer than the //! record retention window plus the corrupt leftovers, `--all` -//! removes every record (only the last five per target are kept by -//! `pkh deb` itself). +//! removes every record (at most five staged trees per target are +//! kept by `pkh deb` itself). +//! - **Shared apt download caches** under `/var/tmp/pkh/apt-cache/` +//! (see [`crate::deb::ephemeral::apt_cache_dir`]): the default policy +//! removes the ones untouched for longer than the retention window, +//! `--all` removes them all; the `.debs` they hold are cheap to +//! re-download. //! - **Cached chroot tarballs** (`~/.cache/pkh/*-buildd.tar.xz`) and their //! **stale download lockfiles** (`~/.cache/pkh/*.lock` untouched for longer //! than [`LOCK_STALE_AFTER`]; younger lockfiles may belong to a concurrent @@ -103,6 +109,9 @@ enum Artifact { Session(PathBuf), /// A `pkh test` run directory (report artifacts). TestRun(PathBuf), + /// A shared apt download cache of one target (see + /// [`crate::deb::ephemeral::apt_cache_dir`]). + AptCache(PathBuf), /// An orphaned record lock file (`.lock` whose record /// root is gone). SessionLock(PathBuf), @@ -118,6 +127,7 @@ impl Artifact { | Artifact::LogFile(p) | Artifact::Session(p) | Artifact::TestRun(p) + | Artifact::AptCache(p) | Artifact::SessionLock(p) => p, } } @@ -131,6 +141,7 @@ impl Artifact { Artifact::LogFile(_) => "build log", Artifact::Session(_) => "build record", Artifact::TestRun(_) => "pkh test run", + Artifact::AptCache(_) => "shared apt download cache", Artifact::SessionLock(_) => "stale record lockfile", } } @@ -231,6 +242,25 @@ fn discover_test_runs(tests_dir: &Path, all: bool) -> Vec { .collect() } +/// Discover the shared apt download caches prune may remove under +/// `apt_cache_dir` (`/var/tmp/pkh/apt-cache`, one `-` +/// subdirectory per target): with `--all`, every one; otherwise only +/// those untouched for longer than the retention window. A cache in use +/// by a live build is fresh by construction — its `.debs` are being +/// read and written. +fn discover_apt_cache(apt_cache_dir: &Path, all: bool) -> Vec { + let Ok(entries) = fs::read_dir(apt_cache_dir) else { + return Vec::new(); + }; + entries + .flatten() + .map(|entry| entry.path()) + .filter(|path| path.is_dir()) + .filter(|path| is_stale_path(path, all)) + .map(Artifact::AptCache) + .collect() +} + /// Whether a record root is prunable under the given policy: everything /// with `--all`, otherwise stale (untouched beyond the retention window) /// or corrupt (no readable manifest) records. Live records record a fresh @@ -582,19 +612,21 @@ pub fn prune_in( cache_dir: Option<&Path>, options: PruneOptions, ) -> Result> { - prune_in_roots(temp_dir, cache_dir, None, None, options) + prune_in_roots(temp_dir, cache_dir, None, None, None, options) } /// Execute the prune operation against explicit roots. /// -/// `sessions_dir` opts in to build-session pruning and `tests_dir` to -/// test-run pruning (the production [`prune()`] passes the real roots; -/// tests pass controlled directories or `None`). +/// `sessions_dir` opts in to build-record pruning, `tests_dir` to +/// test-run pruning and `apt_cache_dir` to shared apt-cache pruning (the +/// production [`prune()`] passes the real roots; tests pass controlled +/// directories or `None`). pub fn prune_in_roots( temp_dir: &Path, cache_dir: Option<&Path>, sessions_dir: Option<&Path>, tests_dir: Option<&Path>, + apt_cache_dir: Option<&Path>, options: PruneOptions, ) -> Result> { let mut artifacts = discover_artifacts(temp_dir, cache_dir); @@ -604,6 +636,9 @@ pub fn prune_in_roots( if let Some(tests_dir) = tests_dir { artifacts.extend(discover_test_runs(tests_dir, options.all)); } + if let Some(apt_cache_dir) = apt_cache_dir { + artifacts.extend(discover_apt_cache(apt_cache_dir, options.all)); + } let mut report = PruneReport { dry_run: options.dry_run, @@ -671,6 +706,7 @@ pub fn prune(options: PruneOptions) -> Result logs += 1, Artifact::Session(_) => {} Artifact::TestRun(_) => {} + Artifact::AptCache(_) => {} Artifact::SessionLock(_) => {} } } @@ -1307,6 +1344,7 @@ none /tmp/other proc rw 0 0 None, Some(&sessions), None, + None, PruneOptions::default(), ) .unwrap(); @@ -1322,6 +1360,7 @@ none /tmp/other proc rw 0 0 None, Some(&sessions), None, + None, PruneOptions { all: true, ..Default::default() @@ -1356,6 +1395,7 @@ none /tmp/other proc rw 0 0 None, None, Some(&tests), + None, PruneOptions::default(), ) .unwrap(); @@ -1369,6 +1409,59 @@ none /tmp/other proc rw 0 0 None, None, Some(&tests), + None, + PruneOptions { + all: true, + ..Default::default() + }, + ) + .unwrap(); + assert!(report.removed.contains(&fresh)); + assert!(!fresh.exists()); + } + + /// The shared apt download caches follow the retention window: a + /// cache untouched for over a week goes, one in recent use stays, + /// `--all` takes everything. + #[test] + fn prunes_stale_apt_caches_by_retention() { + let temp = tempdir().unwrap(); + let apt_cache = temp.path().join("apt-cache"); + let stale = apt_cache.join("noble-amd64"); + let fresh = apt_cache.join("sid-riscv64"); + for dir in [&stale, &fresh] { + fs::create_dir_all(dir.join("partial")).unwrap(); + fs::write(dir.join("some-deb_1.0_amd64.deb"), "deb").unwrap(); + } + // Directories open read-only (write-open is a file-only mode): + // futimens still applies through the read handle. + let dir = fs::File::open(&stale).unwrap(); + dir.set_times( + std::fs::FileTimes::new() + .set_modified(SystemTime::now() - std::time::Duration::from_secs(8 * 24 * 60 * 60)), + ) + .unwrap(); + + let report = prune_in_roots( + temp.path(), + None, + None, + None, + Some(&apt_cache), + PruneOptions::default(), + ) + .unwrap(); + assert!(report.removed.contains(&stale)); + assert!(!stale.exists()); + assert!(fresh.exists(), "a recently used cache is not pruned"); + + // --all removes every cache. + let report = prune_in_roots( + temp.path(), + None, + None, + None, + Some(&apt_cache), PruneOptions { all: true, ..Default::default() @@ -1415,6 +1508,7 @@ none /tmp/other proc rw 0 0 None, Some(&sessions), None, + None, PruneOptions::default(), ) .unwrap(); @@ -1432,6 +1526,7 @@ none /tmp/other proc rw 0 0 None, Some(&sessions), None, + None, PruneOptions { all: true, ..Default::default()