deb: share the apt download cache across builds
With the chroot now per-run scratch, every build re-downloaded the packages its dependency install needed — the very cost the chroot reuse used to hide. Every chroot's /var/cache/apt/archives is now a bind mount over a shared cache at /var/tmp/pkh/apt-cache/<series>-<arch>/ (the chroot's architecture, like the tarball cache), which one run's installs fill and the next runs reuse: apt keeps downloaded packages through a dropped conf (APT::Keep-Downloaded-Packages), and an apt-get autoclean at the end of each run drops the debs the archive no longer serves. pkh prune removes caches untouched for a week. The run holds a host-side flock on the cache for its whole lifetime: apt's own archives lock is acquired inside the chroot's pid namespace, where a concurrent build's apt shows the holder as "process 0" — an invisible process it refuses to wait for — so pkh serializes the traffic itself; concurrent builds of the same target wait for each other. Any bind-mount failure falls back to the chroot's own cache with a warning (correct, just slower), and the temp-chroot teardown now unmounts everything under the chroot before rm -rf, so the removal can never reach through the cache bind mount and delete the shared cache itself.
This commit is contained in:
+236
-40
@@ -6,6 +6,7 @@ use crate::report::BuildView;
|
||||
use directories::ProjectDirs;
|
||||
use std::error::Error;
|
||||
use std::fs;
|
||||
use std::os::unix::io::AsRawFd;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::sync::Arc;
|
||||
@@ -238,8 +239,8 @@ fn is_mountpoint(path: &Path) -> bool {
|
||||
}
|
||||
|
||||
/// The cached chroot tarball path for a series/arch, as used by the
|
||||
/// bootstrap: the resume path compares it against the tarball recorded in
|
||||
/// the session manifest.
|
||||
/// bootstrap: the record's fork gate compares the tarball's digest
|
||||
/// against the sha recorded in a base's manifest.
|
||||
pub(crate) fn chroot_tarball_path(
|
||||
series: &str,
|
||||
arch: Option<&str>,
|
||||
@@ -254,6 +255,40 @@ pub(crate) fn chroot_tarball_path(
|
||||
Ok(proj_dirs.cache_dir().join(tarball_filename))
|
||||
}
|
||||
|
||||
/// Root of the shared apt download caches (`/var/tmp/pkh/apt-cache`),
|
||||
/// one `<series>-<arch>` subdirectory per target.
|
||||
pub(crate) const APT_CACHE_ROOT: &str = "/var/tmp/pkh/apt-cache";
|
||||
|
||||
/// The shared apt download cache of a `series`/`arch` target:
|
||||
/// `/var/tmp/pkh/apt-cache/<series>-<arch>/`, bind-mounted over each
|
||||
/// chroot's `/var/cache/apt/archives` so dependency installs reuse the
|
||||
/// `.debs` previous runs downloaded instead of fetching them again.
|
||||
/// `arch` is the chroot's architecture (the foreign one for qemu-binfmt
|
||||
/// builds, the host one otherwise), keyed like the chroot tarball cache.
|
||||
pub(crate) fn apt_cache_dir(series: &str, arch: Option<&str>) -> PathBuf {
|
||||
// The host architecture when the chroot is not a foreign one.
|
||||
let arch = arch
|
||||
.map(str::to_string)
|
||||
.unwrap_or_else(crate::get_current_arch);
|
||||
PathBuf::from(APT_CACHE_ROOT).join(format!("{series}-{arch}"))
|
||||
}
|
||||
|
||||
/// Name of the apt configuration file that keeps downloaded packages
|
||||
/// (without it apt deletes the `.debs` it installs, and the shared
|
||||
/// cache would never fill).
|
||||
pub(crate) const APT_KEEP_CONF: &str = "99pkh-shared-download-cache";
|
||||
|
||||
/// Write the apt configuration keeping downloaded packages into the
|
||||
/// chroot, so installs populate the (shared) download cache.
|
||||
pub(crate) fn write_apt_keep_conf(chroot_path: &Path) -> std::io::Result<()> {
|
||||
let conf_dir = chroot_path.join("etc/apt/apt.conf.d");
|
||||
fs::create_dir_all(&conf_dir)?;
|
||||
fs::write(
|
||||
conf_dir.join(APT_KEEP_CONF),
|
||||
"APT::Keep-Downloaded-Packages \"true\";\n",
|
||||
)
|
||||
}
|
||||
|
||||
/// Unmount `path`, falling back to a lazy unmount if the first attempt fails
|
||||
/// because something still holds the mount busy (e.g. an interrupted child
|
||||
/// that has not exited yet); returns whether the mount is gone
|
||||
@@ -349,6 +384,12 @@ pub struct EphemeralContextGuard {
|
||||
/// Whether the record's staged tree is retained after the build
|
||||
/// (`pkh deb --incremental`).
|
||||
keep_upper: bool,
|
||||
/// The host-side lock of the shared apt download cache, held while
|
||||
/// its bind mount is up (see
|
||||
/// [`EphemeralContextGuard::share_apt_cache`]); released when the
|
||||
/// guard drops, together with the mount. Held for its drop side
|
||||
/// effect alone, hence the underscore: it is never read.
|
||||
_apt_cache_lock: Option<fs::File>,
|
||||
}
|
||||
|
||||
impl EphemeralContextGuard {
|
||||
@@ -456,25 +497,34 @@ impl EphemeralContextGuard {
|
||||
};
|
||||
|
||||
// Download and extract the chroot tarball — always fresh; the
|
||||
// record's chroot directory is per-run scratch.
|
||||
if let Err(e) =
|
||||
Self::download_and_extract_chroot(series, arch, &chroot_path, base_ctx.clone(), view)
|
||||
.await
|
||||
// record's chroot directory is per-run scratch. The shared apt
|
||||
// cache's lock (when the cache is shared) is held from here on.
|
||||
let apt_cache_lock = match Self::download_and_extract_chroot(
|
||||
series,
|
||||
arch,
|
||||
&chroot_path,
|
||||
base_ctx.clone(),
|
||||
view,
|
||||
)
|
||||
.await
|
||||
{
|
||||
// On a Ctrl+C the interrupt watchdog owns the tree: keep the
|
||||
// hook registered (forgetting the guard) so it removes the
|
||||
// partial directory, instead of the historical behavior of
|
||||
// leaving it in place. Without an interrupt this is a plain
|
||||
// bootstrap failure and the partial directory stays, as before.
|
||||
if crate::interrupt::interrupted()
|
||||
&& let Some(hook) = cleanup_hook
|
||||
{
|
||||
std::mem::forget(hook);
|
||||
} else {
|
||||
drop(cleanup_hook);
|
||||
Ok(lock) => lock,
|
||||
Err(e) => {
|
||||
// On a Ctrl+C the interrupt watchdog owns the tree: keep the
|
||||
// hook registered (forgetting the guard) so it removes the
|
||||
// partial directory, instead of the historical behavior of
|
||||
// leaving it in place. Without an interrupt this is a plain
|
||||
// bootstrap failure and the partial directory stays, as before.
|
||||
if crate::interrupt::interrupted()
|
||||
&& let Some(hook) = cleanup_hook
|
||||
{
|
||||
std::mem::forget(hook);
|
||||
} else {
|
||||
drop(cleanup_hook);
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
|
||||
// Record the tarball digest this environment came from: the fork
|
||||
// gate of a later `--incremental` run compares it against the
|
||||
@@ -514,6 +564,7 @@ impl EphemeralContextGuard {
|
||||
cleanup_hook,
|
||||
session,
|
||||
keep_upper,
|
||||
_apt_cache_lock: apt_cache_lock,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -533,7 +584,7 @@ impl EphemeralContextGuard {
|
||||
chroot_path: &PathBuf,
|
||||
ctx: Arc<context::Context>,
|
||||
view: &dyn BuildView,
|
||||
) -> Result<(), Box<dyn Error>> {
|
||||
) -> Result<Option<fs::File>, Box<dyn Error>> {
|
||||
// The chroot is per-run scratch: nothing is reused, so never
|
||||
// extract over stale content.
|
||||
clear_dir(chroot_path)?;
|
||||
@@ -612,21 +663,26 @@ impl EphemeralContextGuard {
|
||||
enter_phase(view, Phase::ExtractingChroot);
|
||||
Self::extract_tarball(&tarball_path, chroot_path, view)?;
|
||||
|
||||
// Create device nodes in the chroot and bind-mount /proc
|
||||
Self::finalize_chroot(chroot_path, ctx_for_devices, view)?;
|
||||
// Create device nodes, bind-mount /proc and share the apt cache
|
||||
let apt_cache_lock =
|
||||
Self::finalize_chroot(series, arch, chroot_path, ctx_for_devices, view)?;
|
||||
|
||||
// The bootstrap's tarball digest is recorded by the caller (the
|
||||
// record's fork gate compares it against the cache later).
|
||||
Ok(())
|
||||
Ok(apt_cache_lock)
|
||||
}
|
||||
|
||||
/// Device nodes and the /proc bind mount for a fresh chroot:
|
||||
/// idempotent finalization, safe to run on every run.
|
||||
/// Device nodes, the /proc bind mount and the shared apt download
|
||||
/// cache for a fresh chroot: idempotent finalization, safe to run on
|
||||
/// every run. Returns the shared cache's lock, held while the mount
|
||||
/// is up (see [`EphemeralContextGuard::share_apt_cache`]).
|
||||
fn finalize_chroot(
|
||||
series: &str,
|
||||
arch: Option<&str>,
|
||||
chroot_path: &Path,
|
||||
ctx: Arc<context::Context>,
|
||||
view: &dyn BuildView,
|
||||
) -> Result<(), Box<dyn Error>> {
|
||||
) -> Result<Option<fs::File>, Box<dyn Error>> {
|
||||
if !chroot_path.join("dev/null").exists() {
|
||||
// Create device nodes in the chroot
|
||||
log::debug!("Creating device nodes in chroot...");
|
||||
@@ -642,9 +698,112 @@ impl EphemeralContextGuard {
|
||||
log::debug!("/proc is already bind-mounted in the chroot");
|
||||
} else {
|
||||
log::debug!("Bind-mounting /proc into chroot...");
|
||||
Self::bind_mount_proc(chroot_path, ctx)?;
|
||||
Self::bind_mount_proc(chroot_path, ctx.clone())?;
|
||||
}
|
||||
|
||||
Ok(Self::share_apt_cache(series, arch, chroot_path, ctx))
|
||||
}
|
||||
|
||||
/// Bind-mount the shared apt download cache of `series`/`arch` over
|
||||
/// the chroot's `/var/cache/apt/archives`, so dependency installs
|
||||
/// reuse the `.debs` previous runs downloaded instead of fetching
|
||||
/// them again. The apt conf keeping downloaded packages
|
||||
/// ([`write_apt_keep_conf`]) makes the installs populate it.
|
||||
///
|
||||
/// While the mount is up, the run holds a host-side flock on the
|
||||
/// cache's lock file (returned to the caller, released when the
|
||||
/// guard drops): apt's own `archives/lock` is acquired inside the
|
||||
/// chroot's pid namespace, where a concurrent build's apt appears
|
||||
/// as "held by process 0" — an invisible process it refuses to wait
|
||||
/// for — so pkh serializes the apt traffic of builds sharing one
|
||||
/// cache itself. Concurrent builds of the same target wait for each
|
||||
/// other instead of racing apt.
|
||||
///
|
||||
/// Any failure is a warning, never an error: the chroot keeps its
|
||||
/// own cache dir and everything stays correct, just slower.
|
||||
fn share_apt_cache(
|
||||
series: &str,
|
||||
arch: Option<&str>,
|
||||
chroot_path: &Path,
|
||||
ctx: Arc<context::Context>,
|
||||
) -> Option<fs::File> {
|
||||
let cache = apt_cache_dir(series, arch);
|
||||
if let Err(e) = fs::create_dir_all(cache.join("partial")) {
|
||||
log::warn!(
|
||||
"Cannot create the shared apt cache {}: {e}; using the \
|
||||
chroot's own cache",
|
||||
cache.display()
|
||||
);
|
||||
return None;
|
||||
}
|
||||
let archives = chroot_path.join("var/cache/apt/archives");
|
||||
if let Err(e) =
|
||||
fs::create_dir_all(&archives).and_then(|()| write_apt_keep_conf(chroot_path))
|
||||
{
|
||||
log::warn!(
|
||||
"Cannot prepare the chroot's apt cache for sharing: {e}; \
|
||||
using the chroot's own cache"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
// Serialize with the other runs sharing this cache BEFORE apt
|
||||
// can run in the chroot. A held lock is a live run: wait for it
|
||||
// (a crashed pkh releases its flock with its process).
|
||||
let lock_path = cache.with_extension("lock");
|
||||
let lock = match fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&lock_path)
|
||||
{
|
||||
Ok(file) => file,
|
||||
Err(e) => {
|
||||
log::warn!(
|
||||
"Cannot open the shared apt cache lock {}: {e}; using \
|
||||
the chroot's own cache",
|
||||
lock_path.display()
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let taken = unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX) };
|
||||
if taken != 0 {
|
||||
log::warn!(
|
||||
"Cannot lock the shared apt cache {}: {}; using the \
|
||||
chroot's own cache",
|
||||
lock_path.display(),
|
||||
std::io::Error::last_os_error()
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
// Check if we're running as root to avoid unnecessary sudo
|
||||
let is_root = crate::utils::root::is_root().unwrap_or(false);
|
||||
let mut cmd = ctx.command(if is_root { "mount" } else { "sudo" });
|
||||
if !is_root {
|
||||
cmd.arg("mount");
|
||||
}
|
||||
let status = cmd.arg("--bind").arg(&cache).arg(&archives).status();
|
||||
match status {
|
||||
Ok(status) if status.success() => {
|
||||
log::debug!(
|
||||
"Bind-mounted the shared apt cache {} over {}",
|
||||
cache.display(),
|
||||
archives.display()
|
||||
);
|
||||
Some(lock)
|
||||
}
|
||||
outcome => {
|
||||
log::warn!(
|
||||
"Cannot bind-mount the shared apt cache {} over {}: \
|
||||
{:?}; using the chroot's own cache",
|
||||
cache.display(),
|
||||
archives.display(),
|
||||
outcome
|
||||
);
|
||||
None // dropping the lock: nothing is shared
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn download_chroot_tarball(
|
||||
@@ -952,21 +1111,22 @@ impl Drop for EphemeralContextGuard {
|
||||
"Build succeeded, removing chroot directory: {}",
|
||||
self.chroot_path.display()
|
||||
);
|
||||
|
||||
// Check if we're running as root to avoid unnecessary sudo
|
||||
let is_root = crate::utils::root::is_root().unwrap_or(false);
|
||||
|
||||
// Unmount /proc from chroot before removing (ignore errors)
|
||||
let proc_path = self.chroot_path.join("proc");
|
||||
let _ = if is_root {
|
||||
self.base_ctx.command("umount").arg(&proc_path).status()
|
||||
} else {
|
||||
self.base_ctx
|
||||
.command("sudo")
|
||||
.arg("umount")
|
||||
.arg(&proc_path)
|
||||
.status()
|
||||
};
|
||||
// Unmount everything under the chroot before removing it:
|
||||
// /proc, and the shared apt download cache bind-mounted over
|
||||
// /var/cache/apt/archives — an rm -rf through that mount
|
||||
// would delete the shared cache itself.
|
||||
for mount in host_mounts_under(&self.chroot_path).into_iter().rev() {
|
||||
if !unmount_path(&mount, is_root) {
|
||||
log::warn!(
|
||||
"Failed to unmount {} under {}",
|
||||
mount.display(),
|
||||
self.chroot_path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let result = if is_root {
|
||||
self.base_ctx
|
||||
@@ -1156,4 +1316,40 @@ mod chroot_cleanup_tests {
|
||||
let read = std::fs::read_dir(&cleared).unwrap();
|
||||
assert_eq!(read.count(), 0, "the directory must be empty");
|
||||
}
|
||||
|
||||
/// The shared apt cache is keyed per series and chroot architecture
|
||||
/// (the foreign arch for qemu-binfmt builds, the host one
|
||||
/// otherwise), like the chroot tarball cache.
|
||||
#[test]
|
||||
fn apt_cache_dirs_are_keyed_per_series_and_arch() {
|
||||
assert_eq!(
|
||||
apt_cache_dir("noble", Some("riscv64")),
|
||||
PathBuf::from("/var/tmp/pkh/apt-cache/noble-riscv64")
|
||||
);
|
||||
let native = apt_cache_dir("noble", None);
|
||||
assert_eq!(
|
||||
native,
|
||||
PathBuf::from(format!(
|
||||
"/var/tmp/pkh/apt-cache/noble-{}",
|
||||
crate::get_current_arch()
|
||||
))
|
||||
);
|
||||
assert_ne!(native, apt_cache_dir("sid", None), "per series");
|
||||
}
|
||||
|
||||
/// The apt configuration keeping downloaded packages lands in the
|
||||
/// chroot's apt.conf.d, so installs populate the shared cache.
|
||||
#[test]
|
||||
fn apt_keep_conf_lands_in_the_chroot() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let chroot = dir.path().join("chroot");
|
||||
write_apt_keep_conf(&chroot).unwrap();
|
||||
let conf = chroot.join("etc/apt/apt.conf.d").join(APT_KEEP_CONF);
|
||||
let content = fs::read_to_string(&conf).unwrap();
|
||||
assert_eq!(content, "APT::Keep-Downloaded-Packages \"true\";\n");
|
||||
|
||||
// Idempotent: a second write does not fail or duplicate.
|
||||
write_apt_keep_conf(&chroot).unwrap();
|
||||
assert_eq!(fs::read_to_string(&conf).unwrap(), content);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -410,6 +410,23 @@ pub async fn build(
|
||||
s.record_phase("binary", None);
|
||||
}
|
||||
|
||||
// Trim the shared download cache before the chroot goes away: drop
|
||||
// the .debs apt can no longer download (removed from the archive),
|
||||
// keep the rest for the next run of this target. Best-effort — a
|
||||
// failure costs cache space, never the build.
|
||||
let status = cap(
|
||||
ctx.command("apt-get")
|
||||
.envs(env.clone())
|
||||
.arg("-y")
|
||||
.arg("autoclean"),
|
||||
&sink,
|
||||
)
|
||||
.status();
|
||||
match status {
|
||||
Ok(status) if status.success() => {}
|
||||
outcome => log::warn!("apt-get autoclean failed ({outcome:?})"),
|
||||
}
|
||||
|
||||
Ok(artifacts)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user