vhaudiquet/pkh
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

deb: share the apt download cache across builds

Browse Source 
With the chroot now per-run scratch, every build re-downloaded the
packages its dependency install needed — the very cost the chroot reuse
used to hide. Every chroot's /var/cache/apt/archives is now a bind
mount over a shared cache at /var/tmp/pkh/apt-cache/<series>-<arch>/
(the chroot's architecture, like the tarball cache), which one run's
installs fill and the next runs reuse: apt keeps downloaded packages
through a dropped conf (APT::Keep-Downloaded-Packages), and an
apt-get autoclean at the end of each run drops the debs the archive no
longer serves. pkh prune removes caches untouched for a week.

The run holds a host-side flock on the cache for its whole lifetime:
apt's own archives lock is acquired inside the chroot's pid namespace,
where a concurrent build's apt shows the holder as "process 0" — an
invisible process it refuses to wait for — so pkh serializes the
traffic itself; concurrent builds of the same target wait for each
other. Any bind-mount failure falls back to the chroot's own cache
with a warning (correct, just slower), and the temp-chroot teardown now
unmounts everything under the chroot before rm -rf, so the removal can
never reach through the cache bind mount and delete the shared cache
itself.
This commit is contained in:
vhaudiquet
2026-09-29 14:22:49 +02:00
parent dd2634abe3
commit 0e1e37b2e6
4 changed files with 363 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+8
View File
@@ -132,6 +132,14 @@ so records stay small. What `--incremental` buys is the record's
**staged tree** (`tree-upper/`): the overlay upperdir where the build's
writes land, holding the compiled objects.
Fresh chroots do not mean fresh downloads: every chroot's
`/var/cache/apt/archives` is a bind mount over a shared apt download
cache at `/var/tmp/pkh/apt-cache/<series>-<arch>/`, which the installs
of one run fill and the next runs reuse (apt is configured to keep the
downloaded packages, and an `apt-get autoclean` at the end of each run
drops the debs the archive no longer serves). `pkh prune` removes the
caches untouched for a week.
`--incremental` forks a previous record — a copy, the base is never
modified — so a failed run can never destroy the last-good state it was
built from. The staged tree is carried only when the base's recorded
src/deb/ephemeral.rs
+236 -40
View File
@@ -6,6 +6,7 @@ use crate::report::BuildView;
use directories::ProjectDirs;
use std::error::Error;
use std::fs;
use std::os::unix::io::AsRawFd;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::Arc;
@@ -238,8 +239,8 @@ fn is_mountpoint(path: &Path) -> bool {
}
/// The cached chroot tarball path for a series/arch, as used by the
/// bootstrap: the resume path compares it against the tarball recorded in
/// the session manifest.
/// bootstrap: the record's fork gate compares the tarball's digest
/// against the sha recorded in a base's manifest.
pub(crate) fn chroot_tarball_path(
series: &str,
arch: Option<&str>,
@@ -254,6 +255,40 @@ pub(crate) fn chroot_tarball_path(
Ok(proj_dirs.cache_dir().join(tarball_filename))
}
/// Root of the shared apt download caches (`/var/tmp/pkh/apt-cache`),
/// one `<series>-<arch>` subdirectory per target.
pub(crate) const APT_CACHE_ROOT: &str = "/var/tmp/pkh/apt-cache";
/// The shared apt download cache of a `series`/`arch` target:
/// `/var/tmp/pkh/apt-cache/<series>-<arch>/`, bind-mounted over each
/// chroot's `/var/cache/apt/archives` so dependency installs reuse the
/// `.debs` previous runs downloaded instead of fetching them again.
/// `arch` is the chroot's architecture (the foreign one for qemu-binfmt
/// builds, the host one otherwise), keyed like the chroot tarball cache.
pub(crate) fn apt_cache_dir(series: &str, arch: Option<&str>) -> PathBuf {
// The host architecture when the chroot is not a foreign one.
let arch = arch
.map(str::to_string)
.unwrap_or_else(crate::get_current_arch);
PathBuf::from(APT_CACHE_ROOT).join(format!("{series}-{arch}"))
}
/// Name of the apt configuration file that keeps downloaded packages
/// (without it apt deletes the `.debs` it installs, and the shared
/// cache would never fill).
pub(crate) const APT_KEEP_CONF: &str = "99pkh-shared-download-cache";
/// Write the apt configuration keeping downloaded packages into the
/// chroot, so installs populate the (shared) download cache.
pub(crate) fn write_apt_keep_conf(chroot_path: &Path) -> std::io::Result<()> {
let conf_dir = chroot_path.join("etc/apt/apt.conf.d");
fs::create_dir_all(&conf_dir)?;
fs::write(
conf_dir.join(APT_KEEP_CONF),
"APT::Keep-Downloaded-Packages \"true\";\n",
)
}
/// Unmount `path`, falling back to a lazy unmount if the first attempt fails
/// because something still holds the mount busy (e.g. an interrupted child
/// that has not exited yet); returns whether the mount is gone
@@ -349,6 +384,12 @@ pub struct EphemeralContextGuard {
/// Whether the record's staged tree is retained after the build
/// (`pkh deb --incremental`).
keep_upper: bool,
/// The host-side lock of the shared apt download cache, held while
/// its bind mount is up (see
/// [`EphemeralContextGuard::share_apt_cache`]); released when the
/// guard drops, together with the mount. Held for its drop side
/// effect alone, hence the underscore: it is never read.
_apt_cache_lock: Option<fs::File>,
}
impl EphemeralContextGuard {
@@ -456,25 +497,34 @@ impl EphemeralContextGuard {
};
// Download and extract the chroot tarball — always fresh; the
// record's chroot directory is per-run scratch.
if let Err(e) =
Self::download_and_extract_chroot(series, arch, &chroot_path, base_ctx.clone(), view)
.await
// record's chroot directory is per-run scratch. The shared apt
// cache's lock (when the cache is shared) is held from here on.
let apt_cache_lock = match Self::download_and_extract_chroot(
series,
arch,
&chroot_path,
base_ctx.clone(),
view,
)
.await
{
// On a Ctrl+C the interrupt watchdog owns the tree: keep the
// hook registered (forgetting the guard) so it removes the
// partial directory, instead of the historical behavior of
// leaving it in place. Without an interrupt this is a plain
// bootstrap failure and the partial directory stays, as before.
if crate::interrupt::interrupted()
&& let Some(hook) = cleanup_hook
{
std::mem::forget(hook);
} else {
drop(cleanup_hook);
Ok(lock) => lock,
Err(e) => {
// On a Ctrl+C the interrupt watchdog owns the tree: keep the
// hook registered (forgetting the guard) so it removes the
// partial directory, instead of the historical behavior of
// leaving it in place. Without an interrupt this is a plain
// bootstrap failure and the partial directory stays, as before.
if crate::interrupt::interrupted()
&& let Some(hook) = cleanup_hook
{
std::mem::forget(hook);
} else {
drop(cleanup_hook);
}
return Err(e);
}
return Err(e);
}
};
// Record the tarball digest this environment came from: the fork
// gate of a later `--incremental` run compares it against the
@@ -514,6 +564,7 @@ impl EphemeralContextGuard {
cleanup_hook,
session,
keep_upper,
_apt_cache_lock: apt_cache_lock,
})
}
@@ -533,7 +584,7 @@ impl EphemeralContextGuard {
chroot_path: &PathBuf,
ctx: Arc<context::Context>,
view: &dyn BuildView,
) -> Result<(), Box<dyn Error>> {
) -> Result<Option<fs::File>, Box<dyn Error>> {
// The chroot is per-run scratch: nothing is reused, so never
// extract over stale content.
clear_dir(chroot_path)?;
@@ -612,21 +663,26 @@ impl EphemeralContextGuard {
enter_phase(view, Phase::ExtractingChroot);
Self::extract_tarball(&tarball_path, chroot_path, view)?;
// Create device nodes in the chroot and bind-mount /proc
Self::finalize_chroot(chroot_path, ctx_for_devices, view)?;
// Create device nodes, bind-mount /proc and share the apt cache
let apt_cache_lock =
Self::finalize_chroot(series, arch, chroot_path, ctx_for_devices, view)?;
// The bootstrap's tarball digest is recorded by the caller (the
// record's fork gate compares it against the cache later).
Ok(())
Ok(apt_cache_lock)
}
/// Device nodes and the /proc bind mount for a fresh chroot:
/// idempotent finalization, safe to run on every run.
/// Device nodes, the /proc bind mount and the shared apt download
/// cache for a fresh chroot: idempotent finalization, safe to run on
/// every run. Returns the shared cache's lock, held while the mount
/// is up (see [`EphemeralContextGuard::share_apt_cache`]).
fn finalize_chroot(
series: &str,
arch: Option<&str>,
chroot_path: &Path,
ctx: Arc<context::Context>,
view: &dyn BuildView,
) -> Result<(), Box<dyn Error>> {
) -> Result<Option<fs::File>, Box<dyn Error>> {
if !chroot_path.join("dev/null").exists() {
// Create device nodes in the chroot
log::debug!("Creating device nodes in chroot...");
@@ -642,9 +698,112 @@ impl EphemeralContextGuard {
log::debug!("/proc is already bind-mounted in the chroot");
} else {
log::debug!("Bind-mounting /proc into chroot...");
Self::bind_mount_proc(chroot_path, ctx)?;
Self::bind_mount_proc(chroot_path, ctx.clone())?;
}
Ok(Self::share_apt_cache(series, arch, chroot_path, ctx))
}
/// Bind-mount the shared apt download cache of `series`/`arch` over
/// the chroot's `/var/cache/apt/archives`, so dependency installs
/// reuse the `.debs` previous runs downloaded instead of fetching
/// them again. The apt conf keeping downloaded packages
/// ([`write_apt_keep_conf`]) makes the installs populate it.
///
/// While the mount is up, the run holds a host-side flock on the
/// cache's lock file (returned to the caller, released when the
/// guard drops): apt's own `archives/lock` is acquired inside the
/// chroot's pid namespace, where a concurrent build's apt appears
/// as "held by process 0" — an invisible process it refuses to wait
/// for — so pkh serializes the apt traffic of builds sharing one
/// cache itself. Concurrent builds of the same target wait for each
/// other instead of racing apt.
///
/// Any failure is a warning, never an error: the chroot keeps its
/// own cache dir and everything stays correct, just slower.
fn share_apt_cache(
series: &str,
arch: Option<&str>,
chroot_path: &Path,
ctx: Arc<context::Context>,
) -> Option<fs::File> {
let cache = apt_cache_dir(series, arch);
if let Err(e) = fs::create_dir_all(cache.join("partial")) {
log::warn!(
"Cannot create the shared apt cache {}: {e}; using the \
chroot's own cache",
cache.display()
);
return None;
}
let archives = chroot_path.join("var/cache/apt/archives");
if let Err(e) =
fs::create_dir_all(&archives).and_then(|()| write_apt_keep_conf(chroot_path))
{
log::warn!(
"Cannot prepare the chroot's apt cache for sharing: {e}; \
using the chroot's own cache"
);
return None;
}
// Serialize with the other runs sharing this cache BEFORE apt
// can run in the chroot. A held lock is a live run: wait for it
// (a crashed pkh releases its flock with its process).
let lock_path = cache.with_extension("lock");
let lock = match fs::OpenOptions::new()
.create(true)
.append(true)
.open(&lock_path)
{
Ok(file) => file,
Err(e) => {
log::warn!(
"Cannot open the shared apt cache lock {}: {e}; using \
the chroot's own cache",
lock_path.display()
);
return None;
}
};
let taken = unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX) };
if taken != 0 {
log::warn!(
"Cannot lock the shared apt cache {}: {}; using the \
chroot's own cache",
lock_path.display(),
std::io::Error::last_os_error()
);
return None;
}
// Check if we're running as root to avoid unnecessary sudo
let is_root = crate::utils::root::is_root().unwrap_or(false);
let mut cmd = ctx.command(if is_root { "mount" } else { "sudo" });
if !is_root {
cmd.arg("mount");
}
let status = cmd.arg("--bind").arg(&cache).arg(&archives).status();
match status {
Ok(status) if status.success() => {
log::debug!(
"Bind-mounted the shared apt cache {} over {}",
cache.display(),
archives.display()
);
Some(lock)
}
outcome => {
log::warn!(
"Cannot bind-mount the shared apt cache {} over {}: \
{:?}; using the chroot's own cache",
cache.display(),
archives.display(),
outcome
);
None // dropping the lock: nothing is shared
}
}
Ok(())
}
async fn download_chroot_tarball(
@@ -952,21 +1111,22 @@ impl Drop for EphemeralContextGuard {
"Build succeeded, removing chroot directory: {}",
self.chroot_path.display()
);
// Check if we're running as root to avoid unnecessary sudo
let is_root = crate::utils::root::is_root().unwrap_or(false);
// Unmount /proc from chroot before removing (ignore errors)
let proc_path = self.chroot_path.join("proc");
let _ = if is_root {
self.base_ctx.command("umount").arg(&proc_path).status()
} else {
self.base_ctx
.command("sudo")
.arg("umount")
.arg(&proc_path)
.status()
};
// Unmount everything under the chroot before removing it:
// /proc, and the shared apt download cache bind-mounted over
// /var/cache/apt/archives — an rm -rf through that mount
// would delete the shared cache itself.
for mount in host_mounts_under(&self.chroot_path).into_iter().rev() {
if !unmount_path(&mount, is_root) {
log::warn!(
"Failed to unmount {} under {}",
mount.display(),
self.chroot_path.display()
);
}
}
let result = if is_root {
self.base_ctx
@@ -1156,4 +1316,40 @@ mod chroot_cleanup_tests {
let read = std::fs::read_dir(&cleared).unwrap();
assert_eq!(read.count(), 0, "the directory must be empty");
}
/// The shared apt cache is keyed per series and chroot architecture
/// (the foreign arch for qemu-binfmt builds, the host one
/// otherwise), like the chroot tarball cache.
#[test]
fn apt_cache_dirs_are_keyed_per_series_and_arch() {
assert_eq!(
apt_cache_dir("noble", Some("riscv64")),
PathBuf::from("/var/tmp/pkh/apt-cache/noble-riscv64")
);
let native = apt_cache_dir("noble", None);
assert_eq!(
native,
PathBuf::from(format!(
"/var/tmp/pkh/apt-cache/noble-{}",
crate::get_current_arch()
))
);
assert_ne!(native, apt_cache_dir("sid", None), "per series");
}
/// The apt configuration keeping downloaded packages lands in the
/// chroot's apt.conf.d, so installs populate the shared cache.
#[test]
fn apt_keep_conf_lands_in_the_chroot() {
let dir = tempfile::tempdir().unwrap();
let chroot = dir.path().join("chroot");
write_apt_keep_conf(&chroot).unwrap();
let conf = chroot.join("etc/apt/apt.conf.d").join(APT_KEEP_CONF);
let content = fs::read_to_string(&conf).unwrap();
assert_eq!(content, "APT::Keep-Downloaded-Packages \"true\";\n");
// Idempotent: a second write does not fail or duplicate.
write_apt_keep_conf(&chroot).unwrap();
assert_eq!(fs::read_to_string(&conf).unwrap(), content);
}
}
src/deb/local.rs
+17
View File
@@ -410,6 +410,23 @@ pub async fn build(
s.record_phase("binary", None);
}
// Trim the shared download cache before the chroot goes away: drop
// the .debs apt can no longer download (removed from the archive),
// keep the rest for the next run of this target. Best-effort — a
// failure costs cache space, never the build.
let status = cap(
ctx.command("apt-get")
.envs(env.clone())
.arg("-y")
.arg("autoclean"),
&sink,
)
.status();
match status {
Ok(status) if status.success() => {}
outcome => log::warn!("apt-get autoclean failed ({outcome:?})"),
}
Ok(artifacts)
}
src/prune.rs
+102 -7
View File
@@ -10,11 +10,17 @@
//! bind-mounted `/proc` and overlay filesystems, so they require careful
//! unmounting before they can be removed.
//! - **Build records** under `/var/tmp/pkh/sessions/<slug>/<record-id>/`
//! (see [`crate::deb::session`]): one immutable record per build; the
//! (see [`crate::deb::session`]): one record per build (metadata and
//! log for every run, the staged tree of `--incremental` ones); the
//! default policy removes the ones untouched for longer than the
//! record retention window plus the corrupt leftovers, `--all`
//! removes every record (only the last five per target are kept by
//! `pkh deb` itself).
//! removes every record (at most five staged trees per target are
//! kept by `pkh deb` itself).
//! - **Shared apt download caches** under `/var/tmp/pkh/apt-cache/`
//! (see [`crate::deb::ephemeral::apt_cache_dir`]): the default policy
//! removes the ones untouched for longer than the retention window,
//! `--all` removes them all; the `.debs` they hold are cheap to
//! re-download.
//! - **Cached chroot tarballs** (`~/.cache/pkh/*-buildd.tar.xz`) and their
//! **stale download lockfiles** (`~/.cache/pkh/*.lock` untouched for longer
//! than [`LOCK_STALE_AFTER`]; younger lockfiles may belong to a concurrent
@@ -103,6 +109,9 @@ enum Artifact {
Session(PathBuf),
/// A `pkh test` run directory (report artifacts).
TestRun(PathBuf),
/// A shared apt download cache of one target (see
/// [`crate::deb::ephemeral::apt_cache_dir`]).
AptCache(PathBuf),
/// An orphaned record lock file (`<record id>.lock` whose record
/// root is gone).
SessionLock(PathBuf),
@@ -118,6 +127,7 @@ impl Artifact {
| Artifact::LogFile(p)
| Artifact::Session(p)
| Artifact::TestRun(p)
| Artifact::AptCache(p)
| Artifact::SessionLock(p) => p,
}
}
@@ -131,6 +141,7 @@ impl Artifact {
Artifact::LogFile(_) => "build log",
Artifact::Session(_) => "build record",
Artifact::TestRun(_) => "pkh test run",
Artifact::AptCache(_) => "shared apt download cache",
Artifact::SessionLock(_) => "stale record lockfile",
}
}
@@ -231,6 +242,25 @@ fn discover_test_runs(tests_dir: &Path, all: bool) -> Vec<Artifact> {
.collect()
}
/// Discover the shared apt download caches prune may remove under
/// `apt_cache_dir` (`/var/tmp/pkh/apt-cache`, one `<series>-<arch>`
/// subdirectory per target): with `--all`, every one; otherwise only
/// those untouched for longer than the retention window. A cache in use
/// by a live build is fresh by construction — its `.debs` are being
/// read and written.
fn discover_apt_cache(apt_cache_dir: &Path, all: bool) -> Vec<Artifact> {
let Ok(entries) = fs::read_dir(apt_cache_dir) else {
return Vec::new();
};
entries
.flatten()
.map(|entry| entry.path())
.filter(|path| path.is_dir())
.filter(|path| is_stale_path(path, all))
.map(Artifact::AptCache)
.collect()
}
/// Whether a record root is prunable under the given policy: everything
/// with `--all`, otherwise stale (untouched beyond the retention window)
/// or corrupt (no readable manifest) records. Live records record a fresh
@@ -582,19 +612,21 @@ pub fn prune_in(
cache_dir: Option<&Path>,
options: PruneOptions,
) -> Result<PruneReport, Box<dyn std::error::Error>> {
prune_in_roots(temp_dir, cache_dir, None, None, options)
prune_in_roots(temp_dir, cache_dir, None, None, None, options)
}
/// Execute the prune operation against explicit roots.
///
/// `sessions_dir` opts in to build-session pruning and `tests_dir` to
/// test-run pruning (the production [`prune()`] passes the real roots;
/// tests pass controlled directories or `None`).
/// `sessions_dir` opts in to build-record pruning, `tests_dir` to
/// test-run pruning and `apt_cache_dir` to shared apt-cache pruning (the
/// production [`prune()`] passes the real roots; tests pass controlled
/// directories or `None`).
pub fn prune_in_roots(
temp_dir: &Path,
cache_dir: Option<&Path>,
sessions_dir: Option<&Path>,
tests_dir: Option<&Path>,
apt_cache_dir: Option<&Path>,
options: PruneOptions,
) -> Result<PruneReport, Box<dyn std::error::Error>> {
let mut artifacts = discover_artifacts(temp_dir, cache_dir);
@@ -604,6 +636,9 @@ pub fn prune_in_roots(
if let Some(tests_dir) = tests_dir {
artifacts.extend(discover_test_runs(tests_dir, options.all));
}
if let Some(apt_cache_dir) = apt_cache_dir {
artifacts.extend(discover_apt_cache(apt_cache_dir, options.all));
}
let mut report = PruneReport {
dry_run: options.dry_run,
@@ -671,6 +706,7 @@ pub fn prune(options: PruneOptions) -> Result<PruneReport, Box<dyn std::error::E
cache.as_deref(),
Some(&crate::deb::session::sessions_root()),
Some(&crate::test::runs_root()),
Some(&PathBuf::from(crate::deb::ephemeral::APT_CACHE_ROOT)),
options,
)
}
@@ -872,6 +908,7 @@ none /tmp/other proc rw 0 0
Artifact::LogFile(_) => logs += 1,
Artifact::Session(_) => {}
Artifact::TestRun(_) => {}
Artifact::AptCache(_) => {}
Artifact::SessionLock(_) => {}
}
}
@@ -1307,6 +1344,7 @@ none /tmp/other proc rw 0 0
None,
Some(&sessions),
None,
None,
PruneOptions::default(),
)
.unwrap();
@@ -1322,6 +1360,7 @@ none /tmp/other proc rw 0 0
None,
Some(&sessions),
None,
None,
PruneOptions {
all: true,
..Default::default()
@@ -1356,6 +1395,7 @@ none /tmp/other proc rw 0 0
None,
None,
Some(&tests),
None,
PruneOptions::default(),
)
.unwrap();
@@ -1369,6 +1409,59 @@ none /tmp/other proc rw 0 0
None,
None,
Some(&tests),
None,
PruneOptions {
all: true,
..Default::default()
},
)
.unwrap();
assert!(report.removed.contains(&fresh));
assert!(!fresh.exists());
}
/// The shared apt download caches follow the retention window: a
/// cache untouched for over a week goes, one in recent use stays,
/// `--all` takes everything.
#[test]
fn prunes_stale_apt_caches_by_retention() {
let temp = tempdir().unwrap();
let apt_cache = temp.path().join("apt-cache");
let stale = apt_cache.join("noble-amd64");
let fresh = apt_cache.join("sid-riscv64");
for dir in [&stale, &fresh] {
fs::create_dir_all(dir.join("partial")).unwrap();
fs::write(dir.join("some-deb_1.0_amd64.deb"), "deb").unwrap();
}
// Directories open read-only (write-open is a file-only mode):
// futimens still applies through the read handle.
let dir = fs::File::open(&stale).unwrap();
dir.set_times(
std::fs::FileTimes::new()
.set_modified(SystemTime::now() - std::time::Duration::from_secs(8 * 24 * 60 * 60)),
)
.unwrap();
let report = prune_in_roots(
temp.path(),
None,
None,
None,
Some(&apt_cache),
PruneOptions::default(),
)
.unwrap();
assert!(report.removed.contains(&stale));
assert!(!stale.exists());
assert!(fresh.exists(), "a recently used cache is not pruned");
// --all removes every cache.
let report = prune_in_roots(
temp.path(),
None,
None,
None,
Some(&apt_cache),
PruneOptions {
all: true,
..Default::default()
@@ -1415,6 +1508,7 @@ none /tmp/other proc rw 0 0
None,
Some(&sessions),
None,
None,
PruneOptions::default(),
)
.unwrap();
@@ -1432,6 +1526,7 @@ none /tmp/other proc rw 0 0
None,
Some(&sessions),
None,
None,
PruneOptions {
all: true,
..Default::default()