diff --git a/src/prune.rs b/src/prune.rs index e0d8cf5..644ef4f 100644 --- a/src/prune.rs +++ b/src/prune.rs @@ -122,107 +122,14 @@ impl PruneGroup { /// Human-readable size for the prune report ("512 B", "3.5 MiB", /// "1.2 GiB", "1.3 TiB"). pub fn format_bytes(bytes: u64) -> String { - const KIB: f64 = 1024.0; - const MIB: f64 = 1024.0 * KIB; - const GIB: f64 = 1024.0 * MIB; - const TIB: f64 = 1024.0 * GIB; - let size = bytes as f64; - if size >= TIB { - format!("{:.1} TiB", size / TIB) - } else if size >= GIB { - format!("{:.1} GiB", size / GIB) - } else if size >= MIB { - format!("{:.1} MiB", size / MIB) - } else if size >= KIB { - format!("{:.1} KiB", size / KIB) - } else { - format!("{bytes} B") - } + crate::utils::size::format_bytes(bytes) } -/// Total size of every artifact in bytes, measured concurrently: the -/// records hold hundreds of thousands of files (a single kernel build -/// tree is ~100 GB of objects), and a serial walk costs tens of seconds. -/// -/// Directory walks never cross a mount point (`du -x`): residual chroots -/// hold bind-mounted `/proc` and overlay filesystems whose virtual files -/// (`/proc/kcore` alone reports 128 TiB) would poison the totals. -/// Unreadable entries count as zero — prune removes them all the same. +/// Total size of every artifact in bytes, measured concurrently (the +/// walker of [`crate::utils::size`]). fn size_artifacts(artifacts: &[&Artifact]) -> Vec { - use std::collections::VecDeque; - use std::os::unix::fs::MetadataExt; - use std::sync::Mutex; - use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; - - // The device each artifact lives on: the walk descends only into - // directories on the same device. - let root_devs: Vec = artifacts - .iter() - .map(|a| fs::symlink_metadata(a.path()).map_or(0, |meta| meta.dev())) - .collect(); - - let totals: Vec = (0..artifacts.len()).map(|_| AtomicU64::new(0)).collect(); - let queue: Mutex> = Mutex::new( - artifacts - .iter() - .enumerate() - .map(|(idx, artifact)| (idx, artifact.path().to_path_buf())) - .collect(), - ); - // Every queued-but-unfinished item: the workers stop when the queue - // is empty and nothing is in flight. - let pending = AtomicUsize::new(artifacts.len()); - - let workers = std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(4) - .min(16); - std::thread::scope(|scope| { - for _ in 0..workers { - scope.spawn(|| { - loop { - let item = queue.lock().unwrap().pop_front(); - let Some((idx, path)) = item else { - if pending.load(Ordering::Acquire) == 0 { - return; - } - std::thread::yield_now(); - continue; - }; - - let Ok(meta) = fs::symlink_metadata(&path) else { - pending.fetch_sub(1, Ordering::AcqRel); - continue; - }; - if !meta.is_dir() { - totals[idx].fetch_add(meta.len(), Ordering::Relaxed); - pending.fetch_sub(1, Ordering::AcqRel); - continue; - } - let Ok(entries) = fs::read_dir(&path) else { - pending.fetch_sub(1, Ordering::AcqRel); - continue; - }; - for entry in entries.flatten() { - let child = entry.path(); - let Ok(meta) = fs::symlink_metadata(&child) else { - continue; - }; - let same_device = meta.dev() == root_devs[idx]; - if meta.is_dir() && same_device { - pending.fetch_add(1, Ordering::AcqRel); - queue.lock().unwrap().push_back((idx, child)); - } else if !meta.is_dir() && same_device { - totals[idx].fetch_add(meta.len(), Ordering::Relaxed); - } - } - pending.fetch_sub(1, Ordering::AcqRel); - } - }); - } - }); - - totals.into_iter().map(|total| total.into_inner()).collect() + let paths: Vec<&Path> = artifacts.iter().map(|a| a.path()).collect(); + crate::utils::size::tree_sizes(&paths) } /// Number of newest build logs kept when `--all` is not passed. diff --git a/src/utils/mod.rs b/src/utils/mod.rs index cbf24ba..a074285 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,2 +1,4 @@ pub mod gpg; pub mod root; +/// Directory-tree sizing (concurrent walker) and byte formatting +pub mod size; diff --git a/src/utils/size.rs b/src/utils/size.rs new file mode 100644 index 0000000..a8b290b --- /dev/null +++ b/src/utils/size.rs @@ -0,0 +1,162 @@ +//! Directory-tree sizing: the concurrent walker shared by the prune +//! report and the staged-tree copy progress, plus the byte formatting +//! both display. + +use std::fs; +use std::path::{Path, PathBuf}; + +/// Total size of every tree in `paths`, in bytes, measured concurrently: +/// the sized trees hold hundreds of thousands of files (a single kernel +/// build tree is ~100 GB of objects), and a serial walk costs tens of +/// seconds. +/// +/// Directory walks never cross a mount point (`du -x`): residual chroots +/// hold bind-mounted `/proc` and overlay filesystems whose virtual files +/// (`/proc/kcore` alone reports 128 TiB) would poison the totals. +/// Unreadable entries count as zero — prune removes them all the same. +pub(crate) fn tree_sizes(paths: &[&Path]) -> Vec { + use std::collections::VecDeque; + use std::os::unix::fs::MetadataExt; + use std::sync::Mutex; + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + + // The device each tree lives on: the walk descends only into + // directories on the same device. + let root_devs: Vec = paths + .iter() + .map(|path| fs::symlink_metadata(path).map_or(0, |meta| meta.dev())) + .collect(); + + let totals: Vec = (0..paths.len()).map(|_| AtomicU64::new(0)).collect(); + let queue: Mutex> = Mutex::new( + paths + .iter() + .enumerate() + .map(|(idx, path)| (idx, path.to_path_buf())) + .collect(), + ); + // Every queued-but-unfinished item: the workers stop when the queue + // is empty and nothing is in flight. + let pending = AtomicUsize::new(paths.len()); + + let workers = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4) + .min(16); + std::thread::scope(|scope| { + for _ in 0..workers { + scope.spawn(|| { + loop { + let item = queue.lock().unwrap().pop_front(); + let Some((idx, path)) = item else { + if pending.load(Ordering::Acquire) == 0 { + return; + } + std::thread::yield_now(); + continue; + }; + + let Ok(meta) = fs::symlink_metadata(&path) else { + pending.fetch_sub(1, Ordering::AcqRel); + continue; + }; + if !meta.is_dir() { + totals[idx].fetch_add(meta.len(), Ordering::Relaxed); + pending.fetch_sub(1, Ordering::AcqRel); + continue; + } + let Ok(entries) = fs::read_dir(&path) else { + pending.fetch_sub(1, Ordering::AcqRel); + continue; + }; + for entry in entries.flatten() { + let child = entry.path(); + let Ok(meta) = fs::symlink_metadata(&child) else { + continue; + }; + let same_device = meta.dev() == root_devs[idx]; + if meta.is_dir() && same_device { + pending.fetch_add(1, Ordering::AcqRel); + queue.lock().unwrap().push_back((idx, child)); + } else if !meta.is_dir() && same_device { + totals[idx].fetch_add(meta.len(), Ordering::Relaxed); + } + } + pending.fetch_sub(1, Ordering::AcqRel); + } + }); + } + }); + + totals.into_iter().map(|total| total.into_inner()).collect() +} + +/// Human-readable size ("512 B", "3.5 MiB", "1.2 GiB", "1.3 TiB"). +pub(crate) fn format_bytes(bytes: u64) -> String { + const KIB: f64 = 1024.0; + const MIB: f64 = 1024.0 * KIB; + const GIB: f64 = 1024.0 * MIB; + const TIB: f64 = 1024.0 * GIB; + let size = bytes as f64; + if size >= TIB { + format!("{:.1} TiB", size / TIB) + } else if size >= GIB { + format!("{:.1} GiB", size / GIB) + } else if size >= MIB { + format!("{:.1} MiB", size / MIB) + } else if size >= KIB { + format!("{:.1} KiB", size / KIB) + } else { + format!("{bytes} B") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use tempfile::tempdir; + + #[test] + fn sizes_files_directories_and_missing_roots() { + let temp = tempdir().unwrap(); + let tree = temp.path().join("tree"); + fs::create_dir_all(tree.join("sub")).unwrap(); + fs::write(tree.join("sub/obj.o"), vec![0u8; 3000]).unwrap(); + fs::write(tree.join("top"), vec![0u8; 10]).unwrap(); + let file = temp.path().join("file"); + fs::write(&file, vec![0u8; 5]).unwrap(); + let missing = temp.path().join("missing"); + + let sizes = tree_sizes(&[tree.as_path(), file.as_path(), missing.as_path()]); + assert_eq!(sizes, vec![3010, 5, 0]); + } + + #[test] + fn symlinks_count_as_themselves_never_their_target() { + let temp = tempdir().unwrap(); + let target = temp.path().join("target"); + fs::write(&target, vec![0u8; 100]).unwrap(); + let link = temp.path().join("link"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + // symlink_metadata: the link's own size (the stored target path), + // never the target's content. + assert_eq!( + tree_sizes(&[link.as_path()]), + vec![target.as_os_str().len() as u64] + ); + } + + #[test] + fn formats_bytes_like_the_reports_display_them() { + assert_eq!(format_bytes(0), "0 B"); + assert_eq!(format_bytes(512), "512 B"); + assert_eq!(format_bytes(2048), "2.0 KiB"); + assert_eq!(format_bytes(5 * 1024 * 1024), "5.0 MiB"); + assert_eq!(format_bytes(3 * 1024 * 1024 * 1024), "3.0 GiB"); + // /proc/kcore inside a bind-mounted chroot /proc reports exactly + // this size; the mount-crossing walk keeps it out of the totals. + assert_eq!(format_bytes(140_737_477_959_680), "128.0 TiB"); + } +}