vhaudiquet/pkh
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

prune,docs: group the prune report by kind with sizes

Browse Source 
The prune output spammed one line per path. The report now groups
entries by artifact kind with their total size, and the size is
measured before removal whatever the mode, so --dry-run prints a
digest ("24 residual build directories for 1.2 GiB") and both modes
end with the freed total. The size walk never crosses a mount point:
residual chroots hold bind-mounted /proc trees whose virtual files
(/proc/kcore alone reports 128 TiB) would poison the total.

--all is also honest now: it removes recent build records and staged
trees (that is where the space is), and the flag description says so.
One behavior fix rides along: a record or lock a live build holds was
previously prunable with --all, which would wreck a running build;
held locks are now out of bounds whatever the policy.
This commit is contained in:
vhaudiquet
2026-10-02 23:22:14 +02:00
parent 8485e7272b
commit 2991237c91
3 changed files with 327 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+4 -1
View File
@@ -185,7 +185,10 @@ pkh deb --no-ccache # build without the shared compile cache
Each target keeps at most five staged trees: when a sixth is retained,
the oldest degrades to a metadata-only record (manifest and log stay),
and `pkh prune` removes records untouched for a week. `pkh deb
and `pkh prune` removes records older than a week. `pkh prune --dry-run`
previews everything it would remove, grouped by kind with sizes;
`--all` ignores the age and removes everything a live build does not
hold, recent records and staged trees included. `pkh deb
--incremental` never forks a record built for a different
series/architecture.
src/main.rs
+16 -21
View File
@@ -549,7 +549,8 @@ fn main() {
clap::Arg::new("all")
.long("all")
.action(clap::ArgAction::SetTrue)
.help("Also remove cached chroot tarballs (expensive to re-download)"),
.help("Remove everything, including recent build records and staged trees")
.long_help("Remove everything, whatever its age: recent build records and their staged trees (the incremental bases `pkh deb --incremental` forks) included, plus the cached chroot tarballs, which are expensive to re-download. Records and locks a live build holds are never touched."),
)
)
.get_matches();
@@ -1208,33 +1209,27 @@ fn main() {
if report.is_empty() {
info!("Nothing to prune.");
} else {
let action = if report.dry_run {
"Would remove"
if report.dry_run {
info!("Dry run - would remove:");
} else {
"Removed"
};
for path in &report.removed {
info!("{} {}", action, path.display());
info!("Removing:");
}
for group in &report.groups {
let size = if group.bytes > 0 {
format!(" for {}", pkh::prune::format_bytes(group.bytes))
} else {
String::new()
};
info!("{} {}{size}", group.count, group.label());
}
for (path, err) in &report.failed {
error!("Failed to remove {}: {}", path.display(), err);
}
let total = pkh::prune::format_bytes(report.total_bytes);
if report.dry_run {
info!(
"(dry run) {} item(s) would be removed.",
report.removed.len()
);
info!("Would have freed {total} total.");
} else {
info!(
"Pruned {} item(s){} ({} failure(s)).",
report.removed.len(),
if all {
" including cached chroot tarballs"
} else {
""
},
report.failed.len()
);
info!("Freed {total} total.");
}
}
if !report.failed.is_empty() {
src/prune.rs
+307 -38
View File
@@ -14,8 +14,10 @@
//! log for every run, the staged tree of `--incremental` ones); the
//! default policy removes the ones untouched for longer than the
//! record retention window plus the corrupt leftovers, `--all`
//! removes every record (at most five staged trees per target are
//! kept by `pkh deb` itself).
//! removes every record — recent builds and staged trees included,
//! which is where the space is. A record a live build holds the lock
//! of is never prunable, whatever the policy (at most five staged
//! trees per target are kept by `pkh deb` itself).
//! - **Shared apt download caches** under `/var/tmp/pkh/apt-cache/`
//! (see [`crate::deb::ephemeral::apt_cache_dir`]): the default policy
//! removes the ones untouched for longer than the retention window,
@@ -39,8 +41,10 @@
//! The [`prune()`] function discovers and removes all of the above. By
//! default it removes everything that is cheap to regenerate (residual
//! directories, keyrings, stale lockfiles, old build logs); pass
//! [`PruneOptions::all`] to also discard the cached chroot tarballs and all
//! build logs, which are expensive to re-create.
//! [`PruneOptions::all`] to also discard the cached chroot tarballs, all
//! build logs and the recent build records. [`PruneOptions::dry_run`]
//! reports the same set grouped by kind with sizes, without removing
//! anything.
use std::fs;
use std::path::{Path, PathBuf};
@@ -68,6 +72,11 @@ pub struct PruneReport {
pub dry_run: bool,
/// Paths that were removed, or that *would* be removed in dry-run mode.
pub removed: Vec<PathBuf>,
/// The removed paths grouped by artifact kind, with sizes. Sizes are
/// measured before removal, whatever the mode.
pub groups: Vec<PruneGroup>,
/// Total size across the groups.
pub total_bytes: u64,
/// Paths that could not be removed, with associated error messages.
pub failed: Vec<(PathBuf, String)>,
}
@@ -79,6 +88,91 @@ impl PruneReport {
}
}
/// One artifact category in a [`PruneReport`]: the artifact kind, how
/// many entries of it were (or would be) removed, and their total size.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PruneGroup {
/// The artifact kind (singular, as discovered).
pub kind: &'static str,
/// How many entries of this kind were removed.
pub count: usize,
/// Total size of the entries in bytes.
pub bytes: u64,
}
impl PruneGroup {
/// The category's display label, pluralized for the count
/// ("1 stale lockfile", "2 stale lockfiles").
pub fn label(&self) -> String {
if self.count == 1 {
return self.kind.to_string();
}
match self.kind.strip_suffix('y') {
Some(stem) => format!("{stem}ies"),
None => format!("{}s", self.kind),
}
}
}
/// Human-readable size for the prune report ("512 B", "3.5 MiB",
/// "1.2 GiB", "1.3 TiB").
pub fn format_bytes(bytes: u64) -> String {
const KIB: f64 = 1024.0;
const MIB: f64 = 1024.0 * KIB;
const GIB: f64 = 1024.0 * MIB;
const TIB: f64 = 1024.0 * GIB;
let size = bytes as f64;
if size >= TIB {
format!("{:.1} TiB", size / TIB)
} else if size >= GIB {
format!("{:.1} GiB", size / GIB)
} else if size >= MIB {
format!("{:.1} MiB", size / MIB)
} else if size >= KIB {
format!("{:.1} KiB", size / KIB)
} else {
format!("{bytes} B")
}
}
/// Total size of `path` in bytes: a file's length, or the recursive walk
/// of a directory. Unreadable entries count as zero — prune removes them
/// all the same. The walk never crosses a mount point (`du -x`): residual
/// chroots hold bind-mounted `/proc` and overlay filesystems whose virtual
/// files (`/proc/kcore` alone reports 128 TiB) would poison the total.
fn path_bytes(path: &Path) -> u64 {
use std::os::unix::fs::MetadataExt;
let Ok(meta) = fs::symlink_metadata(path) else {
return 0;
};
if !meta.is_dir() {
return meta.len();
}
let root_dev = meta.dev();
let mut total = 0;
let mut stack = vec![path.to_path_buf()];
while let Some(dir) = stack.pop() {
let Ok(entries) = fs::read_dir(&dir) else {
continue;
};
for entry in entries.flatten() {
let path = entry.path();
let Ok(meta) = fs::symlink_metadata(&path) else {
continue;
};
if meta.is_dir() {
if meta.dev() == root_dev {
stack.push(path);
}
} else {
total += meta.len();
}
}
}
total
}
/// Number of newest build logs kept when `--all` is not passed.
pub(crate) const KEEP_LOGS: usize = 10;
@@ -150,16 +244,17 @@ impl Artifact {
/// Discover the build records prune may remove under `sessions_dir` (the
/// record layout of [`crate::deb::session`]: one `<record-id>` directory
/// — with its `<record-id>.lock` beside it — per slug directory): with
/// `--all`, every record; otherwise only those untouched for longer than
/// the retention window plus the corrupt leftovers.
/// `--all`, every record (recent builds and staged trees included);
/// otherwise only those untouched for longer than the retention window
/// plus the corrupt leftovers. A record a live build holds the lock of
/// is out of bounds either way.
///
/// Orphaned lock files (`<record id>.lock` whose record root is gone —
/// left behind by interrupted or discarded runs) are collected too: by
/// default only the ones no live build holds (probed with a non-blocking
/// flock), with `--all` every one of them. The pre-record single-session
/// layout kept its lock at the sessions root and its manifest directly
/// inside the slug directory; both are collected as the leftovers they
/// became.
/// left behind by interrupted or discarded runs) are collected too,
/// unless a live build holds them (probed with a non-blocking flock).
/// The pre-record single-session layout kept its lock at the sessions
/// root and its manifest directly inside the slug directory; both are
/// collected as the leftovers they became.
fn discover_sessions(sessions_dir: &Path, all: bool) -> Vec<Artifact> {
let Ok(entries) = fs::read_dir(sessions_dir) else {
return Vec::new();
@@ -172,7 +267,7 @@ fn discover_sessions(sessions_dir: &Path, all: bool) -> Vec<Artifact> {
// slug directory of that name exists.
if let Some(slug) = name.strip_suffix(".lock") {
if path.is_file() && !sessions_dir.join(slug).is_dir() {
collect_lock(&path, all, &mut artifacts);
collect_lock(&path, &mut artifacts);
}
continue;
}
@@ -187,7 +282,7 @@ fn discover_sessions(sessions_dir: &Path, all: bool) -> Vec<Artifact> {
let name = record.file_name().to_string_lossy().into_owned();
if let Some(record_id) = name.strip_suffix(".lock") {
if path.is_file() && !slug_dir.join(record_id).is_dir() {
collect_lock(&path, all, &mut artifacts);
collect_lock(&path, &mut artifacts);
}
} else if path.is_dir() {
if is_prunable_session(&path, all) {
@@ -204,13 +299,14 @@ fn discover_sessions(sessions_dir: &Path, all: bool) -> Vec<Artifact> {
artifacts
}
/// Collect a lock file as prunable when no live build holds it (or `--all`
/// was requested).
fn collect_lock(path: &Path, all: bool, artifacts: &mut Vec<Artifact>) {
let held = crate::deb::session::lock_is_held(path).unwrap_or(true);
if all || !held {
artifacts.push(Artifact::SessionLock(path.to_path_buf()));
/// Collect an orphaned lock file as prunable. A lock a live build holds
/// is never collected, `--all` included: it is mutual exclusion for a
/// concurrent run, not garbage.
fn collect_lock(path: &Path, artifacts: &mut Vec<Artifact>) {
if lock_held(path) {
return;
}
artifacts.push(Artifact::SessionLock(path.to_path_buf()));
}
/// Whether a path is untouched for longer than the record retention
@@ -262,12 +358,18 @@ fn discover_apt_cache(apt_cache_dir: &Path, all: bool) -> Vec<Artifact> {
}
/// Whether a record root is prunable under the given policy: everything
/// with `--all`, otherwise stale (untouched beyond the retention window)
/// or corrupt (no readable manifest) records. Live records record a fresh
/// `last_used` when they are created and are never stale; a record whose
/// manifest is missing is mid-creation rather than corrupt while its
/// lock is held, and is left alone.
/// with `--all` (recent builds and staged trees included — that is where
/// the space is), otherwise stale (untouched beyond the retention window)
/// or corrupt (no readable manifest) records. A record a live build holds
/// the lock of is never prunable, whatever the policy: pruning it would
/// wreck the running build. Live records record a fresh `last_used` when
/// they are created and are never stale; a record whose manifest is
/// missing is mid-creation rather than corrupt while its lock is held,
/// and is left alone.
fn is_prunable_session(path: &Path, all: bool) -> bool {
if lock_held(&record_lock_path(path)) {
return false;
}
if all {
return true;
}
@@ -282,24 +384,32 @@ fn is_prunable_session(path: &Path, all: bool) -> bool {
});
match last_used {
Some(then) => chrono::Utc::now()
.signed_duration_since(then)
.signed_duration_since(chrono::DateTime::<chrono::Utc>::from(then))
.to_std()
.is_ok_and(|age| age >= crate::deb::session::SESSION_RETENTION),
// No readable manifest: a leftover — unless a live build holds
// the record's lock (mid-creation or mid-fork); an unreadable
// probe also errs on keeping it.
None => {
let lock = path.with_file_name(format!(
"{}.lock",
path.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default()
));
matches!(crate::deb::session::lock_is_held(&lock), Ok(false))
}
// No readable manifest: a leftover (the lock is not held, per
// the check above).
None => true,
}
}
/// The advisory lock file of the record root at `path`
/// (`<record-id>.lock` beside it).
fn record_lock_path(path: &Path) -> PathBuf {
path.with_file_name(format!(
"{}.lock",
path.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default()
))
}
/// Whether the lock file at `path` is held by a live build; an unreadable
/// probe errs on "held".
fn lock_held(path: &Path) -> bool {
crate::deb::session::lock_is_held(path).unwrap_or(true)
}
/// Determine the pkh cache directory (e.g. `~/.cache/pkh`), if project dirs
/// can be resolved on this platform.
pub fn cache_dir() -> Option<PathBuf> {
@@ -647,6 +757,9 @@ pub fn prune_in_roots(
let removable_logs = removable_logs(&artifacts, options.all);
// The actionable subset with sizes measured BEFORE anything is
// removed: a removed tree has no size left to report.
let mut actionable: Vec<(&Artifact, u64)> = Vec::new();
for artifact in &artifacts {
// Cached tarballs are only removed when --all is requested: they are
// expensive to re-download.
@@ -661,6 +774,10 @@ pub fn prune_in_roots(
continue;
}
actionable.push((artifact, path_bytes(artifact.path())));
}
for (artifact, _) in &actionable {
let path = artifact.path();
log::info!("{}: {}", artifact.kind(), path.display());
@@ -689,6 +806,25 @@ pub fn prune_in_roots(
}
}
// Group by kind in first-seen order.
for (artifact, bytes) in &actionable {
if let Some(group) = report
.groups
.iter_mut()
.find(|group| group.kind == artifact.kind())
{
group.count += 1;
group.bytes += bytes;
} else {
report.groups.push(PruneGroup {
kind: artifact.kind(),
count: 1,
bytes: *bytes,
});
}
}
report.total_bytes = report.groups.iter().map(|group| group.bytes).sum();
Ok(report)
}
@@ -1371,6 +1507,139 @@ none /tmp/other proc rw 0 0
assert!(!fresh.exists());
}
/// `--all` removes fresh records and staged trees (that is where the
/// space is), but a record whose lock a live build holds is never
/// pruned, and neither is its lock file.
#[test]
fn test_session_pruning_all_spares_records_a_live_build_holds() {
use std::os::unix::io::AsRawFd;
let temp = tempdir().unwrap();
let sessions = temp.path().join("sessions");
let slug = sessions.join("pkh-noble-amd64-1c0ffee0");
let free = slug.join("20260929T000000");
let busy = slug.join("20260929T000001");
write_session_manifest(&free, &chrono::Utc::now().to_rfc3339());
write_session_manifest(&busy, &chrono::Utc::now().to_rfc3339());
// Hold the busy record's lock for the whole test: a live build
// owns it.
let lock_path = slug.join("20260929T000001.lock");
let held = fs::OpenOptions::new()
.create(true)
.truncate(true)
.write(true)
.open(&lock_path)
.unwrap();
unsafe {
libc::flock(held.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB);
}
let report = prune_in_roots(
temp.path(),
None,
Some(&sessions),
None,
None,
PruneOptions::default(),
)
.unwrap();
assert!(report.removed.is_empty(), "fresh records stay by default");
// --all takes the unheld record and leaves the held one (and its
// lock) alone.
let report = prune_in_roots(
temp.path(),
None,
Some(&sessions),
None,
None,
PruneOptions {
all: true,
..Default::default()
},
)
.unwrap();
assert!(report.removed.contains(&free));
assert!(!free.exists());
assert!(busy.exists(), "a held record survives --all");
assert!(lock_path.exists(), "a held lock survives --all");
}
/// The report groups the removed paths by kind with their sizes, so
/// the output is a digest instead of a wall of paths.
#[test]
fn test_prune_report_groups_by_kind_with_sizes() {
let temp = tempdir().unwrap();
let cache = tempdir().unwrap();
let dir = temp.path().join("pkh-1700000000");
fs::create_dir_all(&dir).unwrap();
fs::write(dir.join("obj.o"), vec![0u8; 3000]).unwrap();
let lockfile = cache.path().join("noble-buildd.tar.lock");
fs::write(&lockfile, "lock").unwrap();
set_mtime(&lockfile, old_mtime());
let report = prune_in(
temp.path(),
Some(cache.path()),
PruneOptions {
dry_run: true,
..Default::default()
},
)
.unwrap();
let dirs = report
.groups
.iter()
.find(|g| g.kind == "residual build directory")
.expect("residual directories are grouped");
assert_eq!(dirs.count, 1);
assert_eq!(dirs.bytes, 3000);
// Count 1: the label stays singular; pluralization is covered by
// the labels test below.
assert_eq!(dirs.label(), "residual build directory");
let locks = report
.groups
.iter()
.find(|g| g.kind == "stale lockfile")
.expect("stale lockfiles are grouped");
assert_eq!(locks.count, 1);
assert_eq!(locks.bytes, 4);
assert_eq!(locks.label(), "stale lockfile");
assert_eq!(report.total_bytes, 3004);
assert_eq!(format_bytes(3004), "2.9 KiB");
}
#[test]
fn test_format_bytes_and_group_labels() {
assert_eq!(format_bytes(0), "0 B");
assert_eq!(format_bytes(512), "512 B");
assert_eq!(format_bytes(2048), "2.0 KiB");
assert_eq!(format_bytes(5 * 1024 * 1024), "5.0 MiB");
assert_eq!(format_bytes(3 * 1024 * 1024 * 1024), "3.0 GiB");
// /proc/kcore inside a bind-mounted chroot /proc reports exactly
// this size; the mount-crossing walk keeps it out of the totals.
assert_eq!(format_bytes(140_737_477_959_680), "128.0 TiB");
let group = PruneGroup {
kind: "shared apt download cache",
count: 3,
bytes: 10,
};
assert_eq!(group.label(), "shared apt download caches");
let group = PruneGroup {
kind: "residual build directory",
count: 18,
bytes: 10,
};
assert_eq!(group.label(), "residual build directories");
}
/// `pkh test` run directories follow the retention window: stale
/// runs go, fresh runs stay, `--all` takes everything.
#[test]