diff --git a/README.md b/README.md index ed00239..98b39aa 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,10 @@ pkh deb --no-ccache # build without the shared compile cache Each target keeps at most five staged trees: when a sixth is retained, the oldest degrades to a metadata-only record (manifest and log stay), -and `pkh prune` removes records untouched for a week. `pkh deb +and `pkh prune` removes records older than a week. `pkh prune --dry-run` +previews everything it would remove, grouped by kind with sizes; +`--all` ignores the age and removes everything a live build does not +hold, recent records and staged trees included. `pkh deb --incremental` never forks a record built for a different series/architecture. diff --git a/src/main.rs b/src/main.rs index 1a115ca..5fd5ef5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -549,7 +549,8 @@ fn main() { clap::Arg::new("all") .long("all") .action(clap::ArgAction::SetTrue) - .help("Also remove cached chroot tarballs (expensive to re-download)"), + .help("Remove everything, including recent build records and staged trees") + .long_help("Remove everything, whatever its age: recent build records and their staged trees (the incremental bases `pkh deb --incremental` forks) included, plus the cached chroot tarballs, which are expensive to re-download. Records and locks a live build holds are never touched."), ) ) .get_matches(); @@ -1208,33 +1209,27 @@ fn main() { if report.is_empty() { info!("Nothing to prune."); } else { - let action = if report.dry_run { - "Would remove" + if report.dry_run { + info!("Dry run - would remove:"); } else { - "Removed" - }; - for path in &report.removed { - info!("{} {}", action, path.display()); + info!("Removing:"); + } + for group in &report.groups { + let size = if group.bytes > 0 { + format!(" for {}", pkh::prune::format_bytes(group.bytes)) + } else { + String::new() + }; + info!("{} {}{size}", group.count, group.label()); } for (path, err) in &report.failed { error!("Failed to remove {}: {}", path.display(), err); } + let total = pkh::prune::format_bytes(report.total_bytes); if report.dry_run { - info!( - "(dry run) {} item(s) would be removed.", - report.removed.len() - ); + info!("Would have freed {total} total."); } else { - info!( - "Pruned {} item(s){} ({} failure(s)).", - report.removed.len(), - if all { - " including cached chroot tarballs" - } else { - "" - }, - report.failed.len() - ); + info!("Freed {total} total."); } } if !report.failed.is_empty() { diff --git a/src/prune.rs b/src/prune.rs index 438051d..200b16b 100644 --- a/src/prune.rs +++ b/src/prune.rs @@ -14,8 +14,10 @@ //! log for every run, the staged tree of `--incremental` ones); the //! default policy removes the ones untouched for longer than the //! record retention window plus the corrupt leftovers, `--all` -//! removes every record (at most five staged trees per target are -//! kept by `pkh deb` itself). +//! removes every record — recent builds and staged trees included, +//! which is where the space is. A record a live build holds the lock +//! of is never prunable, whatever the policy (at most five staged +//! trees per target are kept by `pkh deb` itself). //! - **Shared apt download caches** under `/var/tmp/pkh/apt-cache/` //! (see [`crate::deb::ephemeral::apt_cache_dir`]): the default policy //! removes the ones untouched for longer than the retention window, @@ -39,8 +41,10 @@ //! The [`prune()`] function discovers and removes all of the above. By //! default it removes everything that is cheap to regenerate (residual //! directories, keyrings, stale lockfiles, old build logs); pass -//! [`PruneOptions::all`] to also discard the cached chroot tarballs and all -//! build logs, which are expensive to re-create. +//! [`PruneOptions::all`] to also discard the cached chroot tarballs, all +//! build logs and the recent build records. [`PruneOptions::dry_run`] +//! reports the same set grouped by kind with sizes, without removing +//! anything. use std::fs; use std::path::{Path, PathBuf}; @@ -68,6 +72,11 @@ pub struct PruneReport { pub dry_run: bool, /// Paths that were removed, or that *would* be removed in dry-run mode. pub removed: Vec, + /// The removed paths grouped by artifact kind, with sizes. Sizes are + /// measured before removal, whatever the mode. + pub groups: Vec, + /// Total size across the groups. + pub total_bytes: u64, /// Paths that could not be removed, with associated error messages. pub failed: Vec<(PathBuf, String)>, } @@ -79,6 +88,91 @@ impl PruneReport { } } +/// One artifact category in a [`PruneReport`]: the artifact kind, how +/// many entries of it were (or would be) removed, and their total size. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PruneGroup { + /// The artifact kind (singular, as discovered). + pub kind: &'static str, + /// How many entries of this kind were removed. + pub count: usize, + /// Total size of the entries in bytes. + pub bytes: u64, +} + +impl PruneGroup { + /// The category's display label, pluralized for the count + /// ("1 stale lockfile", "2 stale lockfiles"). + pub fn label(&self) -> String { + if self.count == 1 { + return self.kind.to_string(); + } + match self.kind.strip_suffix('y') { + Some(stem) => format!("{stem}ies"), + None => format!("{}s", self.kind), + } + } +} + +/// Human-readable size for the prune report ("512 B", "3.5 MiB", +/// "1.2 GiB", "1.3 TiB"). +pub fn format_bytes(bytes: u64) -> String { + const KIB: f64 = 1024.0; + const MIB: f64 = 1024.0 * KIB; + const GIB: f64 = 1024.0 * MIB; + const TIB: f64 = 1024.0 * GIB; + let size = bytes as f64; + if size >= TIB { + format!("{:.1} TiB", size / TIB) + } else if size >= GIB { + format!("{:.1} GiB", size / GIB) + } else if size >= MIB { + format!("{:.1} MiB", size / MIB) + } else if size >= KIB { + format!("{:.1} KiB", size / KIB) + } else { + format!("{bytes} B") + } +} + +/// Total size of `path` in bytes: a file's length, or the recursive walk +/// of a directory. Unreadable entries count as zero — prune removes them +/// all the same. The walk never crosses a mount point (`du -x`): residual +/// chroots hold bind-mounted `/proc` and overlay filesystems whose virtual +/// files (`/proc/kcore` alone reports 128 TiB) would poison the total. +fn path_bytes(path: &Path) -> u64 { + use std::os::unix::fs::MetadataExt; + + let Ok(meta) = fs::symlink_metadata(path) else { + return 0; + }; + if !meta.is_dir() { + return meta.len(); + } + let root_dev = meta.dev(); + let mut total = 0; + let mut stack = vec![path.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + let Ok(meta) = fs::symlink_metadata(&path) else { + continue; + }; + if meta.is_dir() { + if meta.dev() == root_dev { + stack.push(path); + } + } else { + total += meta.len(); + } + } + } + total +} + /// Number of newest build logs kept when `--all` is not passed. pub(crate) const KEEP_LOGS: usize = 10; @@ -150,16 +244,17 @@ impl Artifact { /// Discover the build records prune may remove under `sessions_dir` (the /// record layout of [`crate::deb::session`]: one `` directory /// — with its `.lock` beside it — per slug directory): with -/// `--all`, every record; otherwise only those untouched for longer than -/// the retention window plus the corrupt leftovers. +/// `--all`, every record (recent builds and staged trees included); +/// otherwise only those untouched for longer than the retention window +/// plus the corrupt leftovers. A record a live build holds the lock of +/// is out of bounds either way. /// /// Orphaned lock files (`.lock` whose record root is gone — -/// left behind by interrupted or discarded runs) are collected too: by -/// default only the ones no live build holds (probed with a non-blocking -/// flock), with `--all` every one of them. The pre-record single-session -/// layout kept its lock at the sessions root and its manifest directly -/// inside the slug directory; both are collected as the leftovers they -/// became. +/// left behind by interrupted or discarded runs) are collected too, +/// unless a live build holds them (probed with a non-blocking flock). +/// The pre-record single-session layout kept its lock at the sessions +/// root and its manifest directly inside the slug directory; both are +/// collected as the leftovers they became. fn discover_sessions(sessions_dir: &Path, all: bool) -> Vec { let Ok(entries) = fs::read_dir(sessions_dir) else { return Vec::new(); @@ -172,7 +267,7 @@ fn discover_sessions(sessions_dir: &Path, all: bool) -> Vec { // slug directory of that name exists. if let Some(slug) = name.strip_suffix(".lock") { if path.is_file() && !sessions_dir.join(slug).is_dir() { - collect_lock(&path, all, &mut artifacts); + collect_lock(&path, &mut artifacts); } continue; } @@ -187,7 +282,7 @@ fn discover_sessions(sessions_dir: &Path, all: bool) -> Vec { let name = record.file_name().to_string_lossy().into_owned(); if let Some(record_id) = name.strip_suffix(".lock") { if path.is_file() && !slug_dir.join(record_id).is_dir() { - collect_lock(&path, all, &mut artifacts); + collect_lock(&path, &mut artifacts); } } else if path.is_dir() { if is_prunable_session(&path, all) { @@ -204,13 +299,14 @@ fn discover_sessions(sessions_dir: &Path, all: bool) -> Vec { artifacts } -/// Collect a lock file as prunable when no live build holds it (or `--all` -/// was requested). -fn collect_lock(path: &Path, all: bool, artifacts: &mut Vec) { - let held = crate::deb::session::lock_is_held(path).unwrap_or(true); - if all || !held { - artifacts.push(Artifact::SessionLock(path.to_path_buf())); +/// Collect an orphaned lock file as prunable. A lock a live build holds +/// is never collected, `--all` included: it is mutual exclusion for a +/// concurrent run, not garbage. +fn collect_lock(path: &Path, artifacts: &mut Vec) { + if lock_held(path) { + return; } + artifacts.push(Artifact::SessionLock(path.to_path_buf())); } /// Whether a path is untouched for longer than the record retention @@ -262,12 +358,18 @@ fn discover_apt_cache(apt_cache_dir: &Path, all: bool) -> Vec { } /// Whether a record root is prunable under the given policy: everything -/// with `--all`, otherwise stale (untouched beyond the retention window) -/// or corrupt (no readable manifest) records. Live records record a fresh -/// `last_used` when they are created and are never stale; a record whose -/// manifest is missing is mid-creation rather than corrupt while its -/// lock is held, and is left alone. +/// with `--all` (recent builds and staged trees included — that is where +/// the space is), otherwise stale (untouched beyond the retention window) +/// or corrupt (no readable manifest) records. A record a live build holds +/// the lock of is never prunable, whatever the policy: pruning it would +/// wreck the running build. Live records record a fresh `last_used` when +/// they are created and are never stale; a record whose manifest is +/// missing is mid-creation rather than corrupt while its lock is held, +/// and is left alone. fn is_prunable_session(path: &Path, all: bool) -> bool { + if lock_held(&record_lock_path(path)) { + return false; + } if all { return true; } @@ -282,24 +384,32 @@ fn is_prunable_session(path: &Path, all: bool) -> bool { }); match last_used { Some(then) => chrono::Utc::now() - .signed_duration_since(then) + .signed_duration_since(chrono::DateTime::::from(then)) .to_std() .is_ok_and(|age| age >= crate::deb::session::SESSION_RETENTION), - // No readable manifest: a leftover — unless a live build holds - // the record's lock (mid-creation or mid-fork); an unreadable - // probe also errs on keeping it. - None => { - let lock = path.with_file_name(format!( - "{}.lock", - path.file_name() - .map(|n| n.to_string_lossy().into_owned()) - .unwrap_or_default() - )); - matches!(crate::deb::session::lock_is_held(&lock), Ok(false)) - } + // No readable manifest: a leftover (the lock is not held, per + // the check above). + None => true, } } +/// The advisory lock file of the record root at `path` +/// (`.lock` beside it). +fn record_lock_path(path: &Path) -> PathBuf { + path.with_file_name(format!( + "{}.lock", + path.file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default() + )) +} + +/// Whether the lock file at `path` is held by a live build; an unreadable +/// probe errs on "held". +fn lock_held(path: &Path) -> bool { + crate::deb::session::lock_is_held(path).unwrap_or(true) +} + /// Determine the pkh cache directory (e.g. `~/.cache/pkh`), if project dirs /// can be resolved on this platform. pub fn cache_dir() -> Option { @@ -647,6 +757,9 @@ pub fn prune_in_roots( let removable_logs = removable_logs(&artifacts, options.all); + // The actionable subset with sizes measured BEFORE anything is + // removed: a removed tree has no size left to report. + let mut actionable: Vec<(&Artifact, u64)> = Vec::new(); for artifact in &artifacts { // Cached tarballs are only removed when --all is requested: they are // expensive to re-download. @@ -661,6 +774,10 @@ pub fn prune_in_roots( continue; } + actionable.push((artifact, path_bytes(artifact.path()))); + } + + for (artifact, _) in &actionable { let path = artifact.path(); log::info!("{}: {}", artifact.kind(), path.display()); @@ -689,6 +806,25 @@ pub fn prune_in_roots( } } + // Group by kind in first-seen order. + for (artifact, bytes) in &actionable { + if let Some(group) = report + .groups + .iter_mut() + .find(|group| group.kind == artifact.kind()) + { + group.count += 1; + group.bytes += bytes; + } else { + report.groups.push(PruneGroup { + kind: artifact.kind(), + count: 1, + bytes: *bytes, + }); + } + } + report.total_bytes = report.groups.iter().map(|group| group.bytes).sum(); + Ok(report) } @@ -1371,6 +1507,139 @@ none /tmp/other proc rw 0 0 assert!(!fresh.exists()); } + /// `--all` removes fresh records and staged trees (that is where the + /// space is), but a record whose lock a live build holds is never + /// pruned, and neither is its lock file. + #[test] + fn test_session_pruning_all_spares_records_a_live_build_holds() { + use std::os::unix::io::AsRawFd; + + let temp = tempdir().unwrap(); + let sessions = temp.path().join("sessions"); + let slug = sessions.join("pkh-noble-amd64-1c0ffee0"); + let free = slug.join("20260929T000000"); + let busy = slug.join("20260929T000001"); + write_session_manifest(&free, &chrono::Utc::now().to_rfc3339()); + write_session_manifest(&busy, &chrono::Utc::now().to_rfc3339()); + + // Hold the busy record's lock for the whole test: a live build + // owns it. + let lock_path = slug.join("20260929T000001.lock"); + let held = fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&lock_path) + .unwrap(); + unsafe { + libc::flock(held.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB); + } + + let report = prune_in_roots( + temp.path(), + None, + Some(&sessions), + None, + None, + PruneOptions::default(), + ) + .unwrap(); + assert!(report.removed.is_empty(), "fresh records stay by default"); + + // --all takes the unheld record and leaves the held one (and its + // lock) alone. + let report = prune_in_roots( + temp.path(), + None, + Some(&sessions), + None, + None, + PruneOptions { + all: true, + ..Default::default() + }, + ) + .unwrap(); + assert!(report.removed.contains(&free)); + assert!(!free.exists()); + assert!(busy.exists(), "a held record survives --all"); + assert!(lock_path.exists(), "a held lock survives --all"); + } + + /// The report groups the removed paths by kind with their sizes, so + /// the output is a digest instead of a wall of paths. + #[test] + fn test_prune_report_groups_by_kind_with_sizes() { + let temp = tempdir().unwrap(); + let cache = tempdir().unwrap(); + + let dir = temp.path().join("pkh-1700000000"); + fs::create_dir_all(&dir).unwrap(); + fs::write(dir.join("obj.o"), vec![0u8; 3000]).unwrap(); + let lockfile = cache.path().join("noble-buildd.tar.lock"); + fs::write(&lockfile, "lock").unwrap(); + set_mtime(&lockfile, old_mtime()); + + let report = prune_in( + temp.path(), + Some(cache.path()), + PruneOptions { + dry_run: true, + ..Default::default() + }, + ) + .unwrap(); + + let dirs = report + .groups + .iter() + .find(|g| g.kind == "residual build directory") + .expect("residual directories are grouped"); + assert_eq!(dirs.count, 1); + assert_eq!(dirs.bytes, 3000); + // Count 1: the label stays singular; pluralization is covered by + // the labels test below. + assert_eq!(dirs.label(), "residual build directory"); + + let locks = report + .groups + .iter() + .find(|g| g.kind == "stale lockfile") + .expect("stale lockfiles are grouped"); + assert_eq!(locks.count, 1); + assert_eq!(locks.bytes, 4); + assert_eq!(locks.label(), "stale lockfile"); + + assert_eq!(report.total_bytes, 3004); + assert_eq!(format_bytes(3004), "2.9 KiB"); + } + + #[test] + fn test_format_bytes_and_group_labels() { + assert_eq!(format_bytes(0), "0 B"); + assert_eq!(format_bytes(512), "512 B"); + assert_eq!(format_bytes(2048), "2.0 KiB"); + assert_eq!(format_bytes(5 * 1024 * 1024), "5.0 MiB"); + assert_eq!(format_bytes(3 * 1024 * 1024 * 1024), "3.0 GiB"); + // /proc/kcore inside a bind-mounted chroot /proc reports exactly + // this size; the mount-crossing walk keeps it out of the totals. + assert_eq!(format_bytes(140_737_477_959_680), "128.0 TiB"); + + let group = PruneGroup { + kind: "shared apt download cache", + count: 3, + bytes: 10, + }; + assert_eq!(group.label(), "shared apt download caches"); + + let group = PruneGroup { + kind: "residual build directory", + count: 18, + bytes: 10, + }; + assert_eq!(group.label(), "residual build directories"); + } + /// `pkh test` run directories follow the retention window: stale /// runs go, fresh runs stay, `--all` takes everything. #[test]