utils/size,prune: share the concurrent tree walker and byte formatting
The incremental fork's staged-tree copy wants to report byte progress against the same measurements the prune report is built from: move the concurrent size walk and the human-readable byte formatting out of prune into utils::size and rewire prune to them. Behavior is unchanged: same-device walks, unreadable entries count as zero.
This commit is contained in:
+5
-98
@@ -122,107 +122,14 @@ impl PruneGroup {
|
||||
/// Human-readable size for the prune report ("512 B", "3.5 MiB",
|
||||
/// "1.2 GiB", "1.3 TiB").
|
||||
pub fn format_bytes(bytes: u64) -> String {
|
||||
const KIB: f64 = 1024.0;
|
||||
const MIB: f64 = 1024.0 * KIB;
|
||||
const GIB: f64 = 1024.0 * MIB;
|
||||
const TIB: f64 = 1024.0 * GIB;
|
||||
let size = bytes as f64;
|
||||
if size >= TIB {
|
||||
format!("{:.1} TiB", size / TIB)
|
||||
} else if size >= GIB {
|
||||
format!("{:.1} GiB", size / GIB)
|
||||
} else if size >= MIB {
|
||||
format!("{:.1} MiB", size / MIB)
|
||||
} else if size >= KIB {
|
||||
format!("{:.1} KiB", size / KIB)
|
||||
} else {
|
||||
format!("{bytes} B")
|
||||
}
|
||||
crate::utils::size::format_bytes(bytes)
|
||||
}
|
||||
|
||||
/// Total size of every artifact in bytes, measured concurrently: the
|
||||
/// records hold hundreds of thousands of files (a single kernel build
|
||||
/// tree is ~100 GB of objects), and a serial walk costs tens of seconds.
|
||||
///
|
||||
/// Directory walks never cross a mount point (`du -x`): residual chroots
|
||||
/// hold bind-mounted `/proc` and overlay filesystems whose virtual files
|
||||
/// (`/proc/kcore` alone reports 128 TiB) would poison the totals.
|
||||
/// Unreadable entries count as zero — prune removes them all the same.
|
||||
/// Total size of every artifact in bytes, measured concurrently (the
|
||||
/// walker of [`crate::utils::size`]).
|
||||
fn size_artifacts(artifacts: &[&Artifact]) -> Vec<u64> {
|
||||
use std::collections::VecDeque;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
|
||||
// The device each artifact lives on: the walk descends only into
|
||||
// directories on the same device.
|
||||
let root_devs: Vec<u64> = artifacts
|
||||
.iter()
|
||||
.map(|a| fs::symlink_metadata(a.path()).map_or(0, |meta| meta.dev()))
|
||||
.collect();
|
||||
|
||||
let totals: Vec<AtomicU64> = (0..artifacts.len()).map(|_| AtomicU64::new(0)).collect();
|
||||
let queue: Mutex<VecDeque<(usize, PathBuf)>> = Mutex::new(
|
||||
artifacts
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, artifact)| (idx, artifact.path().to_path_buf()))
|
||||
.collect(),
|
||||
);
|
||||
// Every queued-but-unfinished item: the workers stop when the queue
|
||||
// is empty and nothing is in flight.
|
||||
let pending = AtomicUsize::new(artifacts.len());
|
||||
|
||||
let workers = std::thread::available_parallelism()
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(4)
|
||||
.min(16);
|
||||
std::thread::scope(|scope| {
|
||||
for _ in 0..workers {
|
||||
scope.spawn(|| {
|
||||
loop {
|
||||
let item = queue.lock().unwrap().pop_front();
|
||||
let Some((idx, path)) = item else {
|
||||
if pending.load(Ordering::Acquire) == 0 {
|
||||
return;
|
||||
}
|
||||
std::thread::yield_now();
|
||||
continue;
|
||||
};
|
||||
|
||||
let Ok(meta) = fs::symlink_metadata(&path) else {
|
||||
pending.fetch_sub(1, Ordering::AcqRel);
|
||||
continue;
|
||||
};
|
||||
if !meta.is_dir() {
|
||||
totals[idx].fetch_add(meta.len(), Ordering::Relaxed);
|
||||
pending.fetch_sub(1, Ordering::AcqRel);
|
||||
continue;
|
||||
}
|
||||
let Ok(entries) = fs::read_dir(&path) else {
|
||||
pending.fetch_sub(1, Ordering::AcqRel);
|
||||
continue;
|
||||
};
|
||||
for entry in entries.flatten() {
|
||||
let child = entry.path();
|
||||
let Ok(meta) = fs::symlink_metadata(&child) else {
|
||||
continue;
|
||||
};
|
||||
let same_device = meta.dev() == root_devs[idx];
|
||||
if meta.is_dir() && same_device {
|
||||
pending.fetch_add(1, Ordering::AcqRel);
|
||||
queue.lock().unwrap().push_back((idx, child));
|
||||
} else if !meta.is_dir() && same_device {
|
||||
totals[idx].fetch_add(meta.len(), Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
pending.fetch_sub(1, Ordering::AcqRel);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
totals.into_iter().map(|total| total.into_inner()).collect()
|
||||
let paths: Vec<&Path> = artifacts.iter().map(|a| a.path()).collect();
|
||||
crate::utils::size::tree_sizes(&paths)
|
||||
}
|
||||
|
||||
/// Number of newest build logs kept when `--all` is not passed.
|
||||
|
||||
Reference in New Issue
Block a user