utils/size,prune: share the concurrent tree walker and byte formatting

The incremental fork's staged-tree copy wants to report byte progress
against the same measurements the prune report is built from: move the
concurrent size walk and the human-readable byte formatting out of
prune into utils::size and rewire prune to them. Behavior is unchanged:
same-device walks, unreadable entries count as zero.
This commit is contained in:
2026-10-03 01:26:45 +02:00
parent 1cc64a37da
commit 83a853d259
3 changed files with 169 additions and 98 deletions
+5 -98
View File
@@ -122,107 +122,14 @@ impl PruneGroup {
/// Human-readable size for the prune report ("512 B", "3.5 MiB",
/// "1.2 GiB", "1.3 TiB").
pub fn format_bytes(bytes: u64) -> String {
const KIB: f64 = 1024.0;
const MIB: f64 = 1024.0 * KIB;
const GIB: f64 = 1024.0 * MIB;
const TIB: f64 = 1024.0 * GIB;
let size = bytes as f64;
if size >= TIB {
format!("{:.1} TiB", size / TIB)
} else if size >= GIB {
format!("{:.1} GiB", size / GIB)
} else if size >= MIB {
format!("{:.1} MiB", size / MIB)
} else if size >= KIB {
format!("{:.1} KiB", size / KIB)
} else {
format!("{bytes} B")
}
crate::utils::size::format_bytes(bytes)
}
/// Total size of every artifact in bytes, measured concurrently: the
/// records hold hundreds of thousands of files (a single kernel build
/// tree is ~100 GB of objects), and a serial walk costs tens of seconds.
///
/// Directory walks never cross a mount point (`du -x`): residual chroots
/// hold bind-mounted `/proc` and overlay filesystems whose virtual files
/// (`/proc/kcore` alone reports 128 TiB) would poison the totals.
/// Unreadable entries count as zero — prune removes them all the same.
/// Total size of every artifact in bytes, measured concurrently (the
/// walker of [`crate::utils::size`]).
fn size_artifacts(artifacts: &[&Artifact]) -> Vec<u64> {
use std::collections::VecDeque;
use std::os::unix::fs::MetadataExt;
use std::sync::Mutex;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
// The device each artifact lives on: the walk descends only into
// directories on the same device.
let root_devs: Vec<u64> = artifacts
.iter()
.map(|a| fs::symlink_metadata(a.path()).map_or(0, |meta| meta.dev()))
.collect();
let totals: Vec<AtomicU64> = (0..artifacts.len()).map(|_| AtomicU64::new(0)).collect();
let queue: Mutex<VecDeque<(usize, PathBuf)>> = Mutex::new(
artifacts
.iter()
.enumerate()
.map(|(idx, artifact)| (idx, artifact.path().to_path_buf()))
.collect(),
);
// Every queued-but-unfinished item: the workers stop when the queue
// is empty and nothing is in flight.
let pending = AtomicUsize::new(artifacts.len());
let workers = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(4)
.min(16);
std::thread::scope(|scope| {
for _ in 0..workers {
scope.spawn(|| {
loop {
let item = queue.lock().unwrap().pop_front();
let Some((idx, path)) = item else {
if pending.load(Ordering::Acquire) == 0 {
return;
}
std::thread::yield_now();
continue;
};
let Ok(meta) = fs::symlink_metadata(&path) else {
pending.fetch_sub(1, Ordering::AcqRel);
continue;
};
if !meta.is_dir() {
totals[idx].fetch_add(meta.len(), Ordering::Relaxed);
pending.fetch_sub(1, Ordering::AcqRel);
continue;
}
let Ok(entries) = fs::read_dir(&path) else {
pending.fetch_sub(1, Ordering::AcqRel);
continue;
};
for entry in entries.flatten() {
let child = entry.path();
let Ok(meta) = fs::symlink_metadata(&child) else {
continue;
};
let same_device = meta.dev() == root_devs[idx];
if meta.is_dir() && same_device {
pending.fetch_add(1, Ordering::AcqRel);
queue.lock().unwrap().push_back((idx, child));
} else if !meta.is_dir() && same_device {
totals[idx].fetch_add(meta.len(), Ordering::Relaxed);
}
}
pending.fetch_sub(1, Ordering::AcqRel);
}
});
}
});
totals.into_iter().map(|total| total.into_inner()).collect()
let paths: Vec<&Path> = artifacts.iter().map(|a| a.path()).collect();
crate::utils::size::tree_sizes(&paths)
}
/// Number of newest build logs kept when `--all` is not passed.