From 6a8c3017058f82a113899bada9006297a05523ff Mon Sep 17 00:00:00 2001 From: Valentin Haudiquet Date: Tue, 29 Sep 2026 14:39:54 +0200 Subject: [PATCH] deb: integrate ccache for content-keyed compile reuse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fresh chroots also mean fresh compiles: nothing survived to make the recompile-heavy middle of a run cheap. Every deb build now wires a shared ccache — /var/tmp/pkh/ccache//, bind-mounted at the chroot's /var/cache/pkh-ccache — through a PATH shim rather than CC environment variables: gcc/g++/cc/c++ symlinks in /usr/lib/pkh-ccache- shim resolve to the ccache the essentials install, so rules that call the plain compiler names are cached while explicit exotic compilers stay untouched. CCACHE_COMPILERCHECK=content is critical — the default is mtime-based and would never hit across fresh chroots, where every toolchain's mtime is the extraction's — with CCACHE_BASEDIR=/tmp/pkh- build and CCACHE_NOHASHDIR=true keying the staged tree's relative paths, and CCACHE_MAXSIZE=10G (PKH_CCACHE_MAXSIZE overrides). No mtime clamping anywhere: the tarball extraction preserves mtimes, so the toolchain and libc headers stay older than carried objects and make keeps them, while build-dependency reinstalls are newer and make rebuilds their dependents — ccache turns those into hits, and a real archive library change is a miss that rebuilds them, exactly the signal clamping would silence. `pkh deb --no-ccache` skips the shim, the env and the mount entirely: an escape hatch for rules that mishandle compiler wrappers (kbuild-style flavour configs). --- README.md | 12 +++- src/deb/ephemeral.rs | 132 ++++++++++++++++++++++++++++++++++++++++--- src/deb/local.rs | 115 +++++++++++++++++++++++++++++++++++++ src/deb/mod.rs | 10 ++++ src/deb/session.rs | 23 ++++++++ src/main.rs | 3 + 6 files changed, 287 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index bcc46a3..ca585c9 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,16 @@ downloaded packages, and an `apt-get autoclean` at the end of each run drops the debs the archive no longer serves). `pkh prune` removes the caches untouched for a week. +Fresh chroots do not mean fresh compiles either: every run's chroot +gets a shared ccache (at `/var/tmp/pkh/ccache//`, mounted at +`/var/cache/pkh-ccache`) wired in through a compiler PATH shim — +`gcc`, `g++`, `cc` and `c++` resolve to ccache — with content-keyed +cache checks (`CCACHE_COMPILERCHECK=content`), so cache hits survive +the fresh toolchain of a new chroot. The cache is capped at 10G per +target (override with `PKH_CCACHE_MAXSIZE`); `pkh deb --no-ccache` +bypasses the shim and cache entirely, an escape hatch for rules that +mishandle compiler wrappers (kbuild-style flavour configs). + `--incremental` forks a previous record — a copy, the base is never modified — so a failed run can never destroy the last-good state it was built from. The staged tree is carried only when the base's recorded @@ -163,7 +173,7 @@ pkh deb list # the records of this tree: id, outcome, # version, fork lineage, staged-tree state, age pkh deb log [-f] [] # print a record's build log; -f follows a # live build -pkh prune # garbage-collect records (7-day retention) +pkh deb --no-ccache # build without the shared compile cache ``` Each target keeps at most five staged trees: when a sixth is retained, diff --git a/src/deb/ephemeral.rs b/src/deb/ephemeral.rs index d798ad7..deff9f8 100644 --- a/src/deb/ephemeral.rs +++ b/src/deb/ephemeral.rs @@ -289,6 +289,82 @@ pub(crate) fn write_apt_keep_conf(chroot_path: &Path) -> std::io::Result<()> { ) } +/// The in-chroot mount point of the shared ccache directory. +pub(crate) const CCACHE_MOUNT: &str = "/var/cache/pkh-ccache"; + +/// The in-chroot directory holding the ccache compiler shims +/// (gcc/g++/cc/c++ → `/usr/bin/ccache`), prepended to the build's PATH +/// by [`crate::deb::local`]. +pub(crate) const CCACHE_SHIM_DIR: &str = "/usr/lib/pkh-ccache-shim"; + +/// Bind-mount the shared ccache directory of a build identity over the +/// chroot's [`CCACHE_MOUNT`] and install the compiler shims, so the +/// builds of that target reuse each other's compiled objects (content +/// keyed, see the ccache env in [`crate::deb::local`]). ccache is +/// designed for concurrent access to one cache — no cross-run lock is +/// needed. +/// +/// Failures are warnings, never errors: an unshared cache dir makes +/// ccache per-run (still correct, just slower), and a missing shim +/// leaves the build on the real compilers. +fn share_ccache(chroot_path: &Path, cache: &Path, ctx: Arc) { + if let Err(e) = fs::create_dir_all(cache).and_then(|()| install_ccache_shim(chroot_path)) { + log::warn!("Cannot share the ccache {}: {e}", cache.display()); + return; + } + let mount_point = chroot_path.join(CCACHE_MOUNT.trim_start_matches('/')); + if let Err(e) = fs::create_dir_all(&mount_point) { + log::warn!( + "Cannot create the ccache mount point {}: {e}", + mount_point.display() + ); + return; + } + // Check if we're running as root to avoid unnecessary sudo + let is_root = crate::utils::root::is_root().unwrap_or(false); + let mut cmd = ctx.command(if is_root { "mount" } else { "sudo" }); + if !is_root { + cmd.arg("mount"); + } + match cmd.arg("--bind").arg(cache).arg(&mount_point).status() { + Ok(status) if status.success() => { + log::debug!( + "Bind-mounted the shared ccache {} over {}", + cache.display(), + mount_point.display() + ); + } + outcome => { + log::warn!( + "Cannot bind-mount the shared ccache {} over {}: {:?}; \ + this run's cache is not shared", + cache.display(), + mount_point.display(), + outcome + ); + } + } +} + +/// Install the ccache PATH shim inside the chroot: `gcc`, `g++`, `cc` +/// and `c++` symlinking to the ccache binary the essentials install, in +/// a directory nothing else populates. The absolute target resolves +/// inside the chroot, where the shim dir is prepended to PATH. +pub(crate) fn install_ccache_shim(chroot_path: &Path) -> std::io::Result<()> { + let shim_dir = chroot_path.join(CCACHE_SHIM_DIR.trim_start_matches('/')); + fs::create_dir_all(&shim_dir)?; + for compiler in ["gcc", "g++", "cc", "c++"] { + let shim = shim_dir.join(compiler); + match fs::symlink_metadata(&shim) { + Ok(_) => fs::remove_file(&shim)?, // idempotent re-install + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + std::os::unix::fs::symlink("/usr/bin/ccache", &shim)?; + } + Ok(()) +} + /// Unmount `path`, falling back to a lazy unmount if the first attempt fails /// because something still holds the mount busy (e.g. an interrupted child /// that has not exited yet); returns whether the mount is gone @@ -406,7 +482,7 @@ impl EphemeralContextGuard { base_ctx: Arc, view: &dyn BuildView, ) -> Result> { - Self::new_internal(None, series, arch, base_ctx, view, false).await + Self::new_internal(None, series, arch, base_ctx, view, false, None).await } /// Create the ephemeral build context inside `session`'s chroot. @@ -421,6 +497,9 @@ impl EphemeralContextGuard { /// * `keep_upper` - Retain the record's staged tree after the build /// (`pkh deb --incremental`); a plain run's record keeps metadata /// and log only + /// * `ccache_dir` - The shared ccache directory to mount at the + /// chroot's `/var/cache/pkh-ccache` (`pkh deb` computes it from the + /// build identity; `None` leaves the chroot unshared) pub async fn new_for_session( session: Session, series: &str, @@ -428,8 +507,18 @@ impl EphemeralContextGuard { base_ctx: Arc, view: &dyn BuildView, keep_upper: bool, + ccache_dir: Option<&Path>, ) -> Result> { - Self::new_internal(Some(session), series, arch, base_ctx, view, keep_upper).await + Self::new_internal( + Some(session), + series, + arch, + base_ctx, + view, + keep_upper, + ccache_dir, + ) + .await } async fn new_internal( @@ -439,6 +528,7 @@ impl EphemeralContextGuard { base_ctx: Arc, view: &dyn BuildView, keep_upper: bool, + ccache_dir: Option<&Path>, ) -> Result> { // Save the globally-installed context so Drop can restore exactly // this handle: concurrent builds install their own ephemeral @@ -505,6 +595,7 @@ impl EphemeralContextGuard { &chroot_path, base_ctx.clone(), view, + ccache_dir, ) .await { @@ -584,6 +675,7 @@ impl EphemeralContextGuard { chroot_path: &PathBuf, ctx: Arc, view: &dyn BuildView, + ccache_dir: Option<&Path>, ) -> Result, Box> { // The chroot is per-run scratch: nothing is reused, so never // extract over stale content. @@ -665,23 +757,25 @@ impl EphemeralContextGuard { // Create device nodes, bind-mount /proc and share the apt cache let apt_cache_lock = - Self::finalize_chroot(series, arch, chroot_path, ctx_for_devices, view)?; + Self::finalize_chroot(series, arch, chroot_path, ctx_for_devices, view, ccache_dir)?; // The bootstrap's tarball digest is recorded by the caller (the // record's fork gate compares it against the cache later). Ok(apt_cache_lock) } - /// Device nodes, the /proc bind mount and the shared apt download - /// cache for a fresh chroot: idempotent finalization, safe to run on - /// every run. Returns the shared cache's lock, held while the mount - /// is up (see [`EphemeralContextGuard::share_apt_cache`]). + /// Device nodes, the /proc bind mount, the shared apt download cache + /// and the shared ccache for a fresh chroot: idempotent + /// finalization, safe to run on every run. Returns the shared apt + /// cache's lock, held while its mount is up (see + /// [`EphemeralContextGuard::share_apt_cache`]). fn finalize_chroot( series: &str, arch: Option<&str>, chroot_path: &Path, ctx: Arc, view: &dyn BuildView, + ccache_dir: Option<&Path>, ) -> Result, Box> { if !chroot_path.join("dev/null").exists() { // Create device nodes in the chroot @@ -701,6 +795,9 @@ impl EphemeralContextGuard { Self::bind_mount_proc(chroot_path, ctx.clone())?; } + if let Some(dir) = ccache_dir { + share_ccache(chroot_path, dir, ctx.clone()); + } Ok(Self::share_apt_cache(series, arch, chroot_path, ctx)) } @@ -1352,4 +1449,25 @@ mod chroot_cleanup_tests { write_apt_keep_conf(&chroot).unwrap(); assert_eq!(fs::read_to_string(&conf).unwrap(), content); } + + /// The ccache shim installs the four compiler symlinks pointing at + /// the ccache binary, and reinstalls cleanly over an existing one. + #[test] + fn ccache_shim_installs_the_four_compiler_links() { + let dir = tempfile::tempdir().unwrap(); + let chroot = dir.path().join("chroot"); + + install_ccache_shim(&chroot).unwrap(); + + let shim_dir = chroot.join(CCACHE_SHIM_DIR.trim_start_matches('/')); + for compiler in ["gcc", "g++", "cc", "c++"] { + let target = fs::read_link(shim_dir.join(compiler)) + .unwrap_or_else(|_| panic!("the {compiler} shim must be a symlink")); + assert_eq!(target, Path::new("/usr/bin/ccache")); + } + + // A second install refreshes the links instead of failing. + install_ccache_shim(&chroot).unwrap(); + assert!(fs::symlink_metadata(shim_dir.join("gcc")).is_ok()); + } } diff --git a/src/deb/local.rs b/src/deb/local.rs index e925d6a..f2f428e 100644 --- a/src/deb/local.rs +++ b/src/deb/local.rs @@ -35,6 +35,58 @@ fn patch_stamp(session: &session::Session) -> Option { session::patches_stamp(&host_tree) } +/// The ccache size cap of a build: `$PKH_CCACHE_MAXSIZE`, default 10G +/// (the unit syntax of ccache's own configuration: `10G`, `512M`, ...). +fn ccache_maxsize() -> String { + std::env::var("PKH_CCACHE_MAXSIZE").unwrap_or_else(|_| "10G".to_string()) +} + +/// Wire the build environment to the shared ccache (on by default, +/// `pkh deb --no-ccache` skips it entirely): the compilers resolve to +/// ccache through a PATH shim, the cache is the identity's shared +/// directory mounted in the chroot, and the cache checks are keyed on +/// content. +/// +/// Mechanics: the tarball extraction preserves mtimes, so the toolchain +/// and libc headers (inside the buildd tarball) stay older than the +/// carried objects and make keeps them; the build dependencies +/// reinstall fresh each run, so their headers are newer and make +/// rebuilds their dependents — ccache turns those rebuilds into hits, +/// and a real archive library change is a cache miss that rebuilds +/// them (the correct, if slow, path). No mtime clamping anywhere: +/// clamping would silence exactly that signal. +/// +/// `CCACHE_COMPILERCHECK=content` is critical: the default is +/// mtime-based and would never hit across fresh chroots, where every +/// toolchain's mtime is the extraction's. +pub(crate) fn apply_ccache_env(env: &mut HashMap, enabled: bool, maxsize: &str) { + if !enabled { + return; + } + // The shim shadows the real compilers with ccache invocations; + // the rest of PATH keeps resolving the other build tools. + let path = env + .get("PATH") + .cloned() + .or_else(|| std::env::var("PATH").ok()) + .unwrap_or_default(); + env.insert( + "PATH".to_string(), + format!("{}:{path}", crate::deb::ephemeral::CCACHE_SHIM_DIR), + ); + env.insert( + "CCACHE_DIR".to_string(), + crate::deb::ephemeral::CCACHE_MOUNT.to_string(), + ); + env.insert("CCACHE_COMPILERCHECK".to_string(), "content".to_string()); + env.insert("CCACHE_MAXSIZE".to_string(), maxsize.to_string()); + env.insert( + "CCACHE_BASEDIR".to_string(), + crate::deb::session::BUILD_ROOT.to_string(), + ); + env.insert("CCACHE_NOHASHDIR".to_string(), "true".to_string()); +} + #[allow(clippy::too_many_arguments)] pub async fn build( package: &str, @@ -52,6 +104,7 @@ pub async fn build( jobs: Option, session: Option<&session::Session>, build_resume: bool, + ccache: bool, ) -> Result, Box> { let sink: Option> = view.sink(); @@ -62,6 +115,7 @@ pub async fn build( let mut env = HashMap::::new(); env.insert("LANG".to_string(), "C".to_string()); env.insert("DEBIAN_FRONTEND".to_string(), "noninteractive".to_string()); + apply_ccache_env(&mut env, ccache, &ccache_maxsize()); // Parallel building: honor an explicit -j/--jobs count, otherwise detect // the number of cores available inside the build context (nproc). @@ -232,6 +286,11 @@ pub async fn build( .arg("build-essential") .arg("dose-builddebcheck") .arg("fakeroot"); + // The compiler cache comes free from the shared download cache; the + // build env wires the compilers to it (see apply_ccache_env). + if ccache { + cmd.arg("ccache"); + } if cross { cmd.arg(format!("crossbuild-essential-{arch}")); cmd.arg(format!("libc6-{arch}-cross")); @@ -1270,6 +1329,62 @@ mod tests { crate::debian::deps::parse_simple(s, true).unwrap() } + /// The build env carries the ccache wiring when it is enabled: the + /// compiler shim first on PATH, the shared cache dir, content-keyed + /// compiler checks and the size cap. + #[test] + fn ccache_env_is_applied_when_enabled() { + let mut env = HashMap::new(); + env.insert("PATH".to_string(), "/usr/bin:/bin".to_string()); + + apply_ccache_env(&mut env, true, "10G"); + + assert_eq!( + env.get("PATH").unwrap(), + &format!("{}:/usr/bin:/bin", crate::deb::ephemeral::CCACHE_SHIM_DIR), + "the shim must shadow the real compilers" + ); + assert_eq!( + env.get("CCACHE_DIR").unwrap(), + crate::deb::ephemeral::CCACHE_MOUNT + ); + assert_eq!( + env.get("CCACHE_COMPILERCHECK").unwrap(), + "content", + "mtime checks would never hit across fresh chroots" + ); + assert_eq!(env.get("CCACHE_MAXSIZE").unwrap(), "10G"); + assert_eq!( + env.get("CCACHE_BASEDIR").unwrap(), + crate::deb::session::BUILD_ROOT + ); + assert_eq!(env.get("CCACHE_NOHASHDIR").unwrap(), "true"); + } + + /// The size cap is the caller's choice: PKH_CCACHE_MAXSIZE flows + /// through as-is. + #[test] + fn ccache_env_honors_the_size_cap() { + let mut env = HashMap::new(); + apply_ccache_env(&mut env, true, "512M"); + assert_eq!(env.get("CCACHE_MAXSIZE").unwrap(), "512M"); + } + + /// With `--no-ccache` the build env is left untouched: no shim, no + /// cache variables. + #[test] + fn ccache_env_is_skipped_when_disabled() { + let mut env = HashMap::new(); + env.insert("PATH".to_string(), "/usr/bin:/bin".to_string()); + + apply_ccache_env(&mut env, false, &ccache_maxsize()); + + assert_eq!(env.get("PATH").unwrap(), "/usr/bin:/bin"); + assert!(!env.contains_key("CCACHE_DIR")); + assert!(!env.contains_key("CCACHE_COMPILERCHECK")); + assert!(!env.contains_key("CCACHE_MAXSIZE")); + } + /// Unqualified dependencies install the dpkg-satisfying variant, /// preferring the runnable build-architecture one: Multi-Arch: /// foreign candidates from the build architecture, then the diff --git a/src/deb/mod.rs b/src/deb/mod.rs index db8d201..e738409 100644 --- a/src/deb/mod.rs +++ b/src/deb/mod.rs @@ -123,6 +123,10 @@ pub struct DebBuildOptions<'a> { /// whether this build's own staged tree is retained. Defaults to a /// plain run. pub incremental: Incremental, + /// Reuse the shared ccache of this target (on by default; `pkh deb + /// --no-ccache` disables it — an escape hatch for `debian/rules` + /// whose own compiler handling conflicts with the shim). + pub ccache: bool, /// Where build events (phases, progress, outcome) are reported. pub view: &'a dyn BuildView, } @@ -215,6 +219,7 @@ impl Default for DebBuildOptions<'_> { jobs: None, ctx: None, incremental: Incremental::Off, + ccache: true, view: &QUIET, } } @@ -258,6 +263,7 @@ async fn build_binary_package_impl( ref jobs, ref ctx, ref incremental, + ccache, view, } = opts; let cwd = cwd.as_deref().unwrap_or_else(|| Path::new(".")); @@ -375,6 +381,8 @@ async fn build_binary_package_impl( None => opts.view, }; + // The shared ccache of this target — skipped with --no-ccache. + let ccache_dir = (session_enabled && ccache).then(|| session::ccache_dir(&identity)); let mut guard = if let Some(s) = session.clone() { Some( ephemeral::EphemeralContextGuard::new_for_session( @@ -384,6 +392,7 @@ async fn build_binary_package_impl( base_ctx.clone(), view, keep_upper, + ccache_dir.as_deref(), ) .await?, ) @@ -522,6 +531,7 @@ async fn build_binary_package_impl( *jobs, session.as_ref(), build_resume, + ccache, ) .await? } diff --git a/src/deb/session.rs b/src/deb/session.rs index 2238277..677d3b0 100644 --- a/src/deb/session.rs +++ b/src/deb/session.rs @@ -707,6 +707,18 @@ pub(crate) fn identity_dir(root: &Path, identity: &SessionIdentity) -> PathBuf { root.join(slug_for(identity)) } +/// Root of the shared ccache directories (`/var/tmp/pkh/ccache`), one +/// per identity slug. +pub(crate) const CCACHE_ROOT: &str = "/var/tmp/pkh/ccache"; + +/// The ccache directory of an identity: `/var/tmp/pkh/ccache//`, +/// bind-mounted at each chroot's `/var/cache/pkh-ccache` so compiled +/// objects are shared across runs (content-keyed, see the ccache setup +/// in `crate::deb::local`). +pub(crate) fn ccache_dir(identity: &SessionIdentity) -> PathBuf { + PathBuf::from(CCACHE_ROOT).join(slug_for(identity)) +} + /// A record id not taken yet under `identity_dir`: the plain timestamp, /// suffixed `-02`, `-03`, ... when earlier builds took the same second /// (zero-padded so the ids keep sorting in creation order). @@ -2187,6 +2199,17 @@ mod tests { assert_ne!(ids[0], ids[1]); } + /// The ccache directory of an identity lives under the shared ccache + /// root, keyed by the identity slug like the records. + #[test] + fn ccache_dir_is_keyed_by_the_identity_slug() { + let identity = test_identity(); + assert_eq!( + ccache_dir(&identity), + PathBuf::from("/var/tmp/pkh/ccache").join(slug_for(&identity)) + ); + } + /// The fork gate: the base's staged tree is carried only while its /// recorded tarball sha still matches the cached tarball AND the /// changelog version is unchanged. Everything else forks cold. diff --git a/src/main.rs b/src/main.rs index cc2a962..684ef89 100644 --- a/src/main.rs +++ b/src/main.rs @@ -374,6 +374,8 @@ fn main() { .help("Incremental build: fork a previous build record of this target and keep its staged tree") .long_help("Incremental build: fork a previous build record of this target — at an unchanged changelog version and while the chroot tarball is unchanged, the base's staged tree is copied into a new record, so make recompiles only what changed.\nWithout a value, the newest successful record of the target (among the last five carrying a staged tree) is the base; with an ID (as shown by 'pkh deb list'), that exact record — a failed one is fine if its staged tree is.\nEvery --incremental run keeps its staged tree, success or failure, so the next one can fork it; a plain 'pkh deb' keeps only metadata and its build log (the chroot is per-run and always discarded)."), ) + .arg(arg!(--"no-ccache" "Build without the shared ccache (escape hatch for kbuild-style rules that mishandle compiler shims)") + .long_help("Build without the shared ccache (its compiler PATH shim is skipped and no cache is mounted). Escape hatch for packages whose debian/rules mishandles a compiler shim, e.g. some kernel flavour configs.")) .arg(arg!(--verbose "Show raw tool output instead of the live build view").required(false) .long_help("Show raw tool output instead of the live build view.\nAlso implied by RUST_LOG=debug for pkh's own logs.")) .subcommand( @@ -1007,6 +1009,7 @@ fn main() { inject, jobs, incremental, + ccache: !sub_matches.get_flag("no-ccache"), view, ..Default::default() })