diff --git a/src/lib.rs b/src/lib.rs index e926df3..288378b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,8 @@ pub mod deb; pub mod distro_info; /// Obtain information about one or multiple packages pub mod package_info; +/// Prune residual pkh build artifacts and caches +pub mod prune; /// Download a source package locally pub mod pull; /// Handle package-specific quirks and workarounds diff --git a/src/main.rs b/src/main.rs index 6e893d7..e0093f9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -101,6 +101,26 @@ fn main() { .arg(arg!( "Context name")) ) ) + .subcommand( + Command::new("prune") + .about("Prune residual pkh build artifacts and caches") + // NOTE: --dry-run is defined via the builder API because clap's + // `arg!` macro mis-tokenizes hyphenated long names (it would + // parse `--dry-run` as long="dry" plus a spurious short flag, + // tripping the "Short flags should precede long flags" assert). + .arg( + clap::Arg::new("dry_run") + .long("dry-run") + .action(clap::ArgAction::SetTrue) + .help("List what would be removed without removing anything"), + ) + .arg( + clap::Arg::new("all") + .long("all") + .action(clap::ArgAction::SetTrue) + .help("Also remove cached chroot tarballs (expensive to re-download)"), + ) + ) .get_matches(); match matches.subcommand() { @@ -359,6 +379,58 @@ fn main() { _ => unreachable!(), } } + Some(("prune", sub_matches)) => { + let dry_run = sub_matches + .get_one::("dry_run") + .copied() + .unwrap_or(false); + let all = sub_matches.get_one::("all").copied().unwrap_or(false); + + let options = pkh::prune::PruneOptions { dry_run, all }; + match pkh::prune::prune(options) { + Ok(report) => { + if report.is_empty() { + info!("Nothing to prune."); + } else { + let action = if report.dry_run { + "Would remove" + } else { + "Removed" + }; + for path in &report.removed { + info!("{} {}", action, path.display()); + } + for (path, err) in &report.failed { + error!("Failed to remove {}: {}", path.display(), err); + } + if report.dry_run { + info!( + "(dry run) {} item(s) would be removed.", + report.removed.len() + ); + } else { + info!( + "Pruned {} item(s){} ({} failure(s)).", + report.removed.len(), + if all { + " including cached chroot tarballs" + } else { + "" + }, + report.failed.len() + ); + } + } + if !report.failed.is_empty() { + std::process::exit(1); + } + } + Err(e) => { + error!("{}", e); + std::process::exit(1); + } + } + } _ => unreachable!("Exhausted list of subcommands and subcommand_required prevents `None`"), } } diff --git a/src/prune.rs b/src/prune.rs new file mode 100644 index 0000000..d0fc287 --- /dev/null +++ b/src/prune.rs @@ -0,0 +1,613 @@ +//! Cleanup of residual pkh build artifacts and caches. +//! +//! `pkh` leaves behind several kinds of artifacts while operating: +//! +//! - **Residual build/chroot directories** under the system temp directory, +//! named `pkh-` (or `pkh--` on collision). +//! They are created by the local and ephemeral build contexts and are +//! *intentionally kept* when a build fails so they can be inspected. They +//! may contain root-owned device nodes (created via `mknod`), a +//! bind-mounted `/proc` and overlay filesystems, so they require careful +//! unmounting before they can be removed. +//! - **Cached chroot tarballs** (`~/.cache/pkh/*-buildd.tar.xz`) and their +//! **stale download lockfiles** (`~/.cache/pkh/*.lock`). +//! - **The shared apt keyring directory** (`/tmp/pkh-keyrings`), used by +//! mmdebstrap runs. +//! +//! The [`prune()`] function discovers and removes all of the above. By +//! default it removes everything that is cheap to regenerate (residual +//! directories, keyrings, stale lockfiles); pass [`PruneOptions::all`] to also +//! discard the cached chroot tarballs, which are expensive to re-download. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use directories::ProjectDirs; + +/// Options controlling the prune operation. +#[derive(Clone, Copy, Default)] +pub struct PruneOptions { + /// When true, do not remove anything; only report what would be removed. + pub dry_run: bool, + /// When true, also remove cached chroot tarballs (which are expensive to + /// re-download). Without this, tarballs are kept and only stale lockfiles + /// are removed from the cache directory. + pub all: bool, +} + +/// Summary of a prune run. +#[derive(Default)] +pub struct PruneReport { + /// Whether this was a dry run (nothing was actually removed). + pub dry_run: bool, + /// Paths that were removed, or that *would* be removed in dry-run mode. + pub removed: Vec, + /// Paths that could not be removed, with associated error messages. + pub failed: Vec<(PathBuf, String)>, +} + +impl PruneReport { + /// Returns true if nothing was removed and nothing failed. + pub fn is_empty(&self) -> bool { + self.removed.is_empty() && self.failed.is_empty() + } +} + +/// One discovered artifact that prune can act on. +#[derive(Debug, Clone)] +enum Artifact { + /// A residual build/chroot directory under the system temp dir. + TempDir(PathBuf), + /// The shared apt keyring directory (`/tmp/pkh-keyrings`). + KeyringDir(PathBuf), + /// A stale chroot tarball download lockfile. + LockFile(PathBuf), + /// A cached chroot tarball. + Tarball(PathBuf), +} + +impl Artifact { + fn path(&self) -> &Path { + match self { + Artifact::TempDir(p) + | Artifact::KeyringDir(p) + | Artifact::LockFile(p) + | Artifact::Tarball(p) => p, + } + } + + fn kind(&self) -> &'static str { + match self { + Artifact::TempDir(_) => "residual build directory", + Artifact::KeyringDir(_) => "apt keyring cache", + Artifact::LockFile(_) => "stale lockfile", + Artifact::Tarball(_) => "cached chroot tarball", + } + } +} + +/// Determine the pkh cache directory (e.g. `~/.cache/pkh`), if project dirs +/// can be resolved on this platform. +pub fn cache_dir() -> Option { + ProjectDirs::from("com", "pkh", "pkh").map(|d| d.cache_dir().to_path_buf()) +} + +/// Check whether a directory entry name is a residual pkh build directory, +/// i.e. it matches `pkh-` or `pkh--`. +/// +/// This deliberately rejects the `pkh-keyrings` and `pkh-build-*` names so they +/// are not mistaken for residual build chroots. +fn is_residual_temp_dir(name: &str) -> bool { + let Some(rest) = name.strip_prefix("pkh-") else { + return false; + }; + let mut parts = rest.splitn(2, '-'); + let first = parts.next().unwrap_or(""); + if first.is_empty() || !first.bytes().all(|b| b.is_ascii_digit()) { + return false; + } + match parts.next() { + None => true, + Some(second) => !second.is_empty() && second.bytes().all(|b| b.is_ascii_digit()), + } +} + +/// Unescape the octal escape sequences used in `/proc/mounts` mount points +/// (e.g. `\040` for space, `\011` for tab, `\012` for newline, `\134` for +/// backslash). +fn unescape_mountpath(s: &str) -> String { + let bytes = s.as_bytes(); + let mut out = String::with_capacity(s.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'\\' && i + 3 < bytes.len() { + let o1 = (bytes[i + 1] as char).to_digit(8); + let o2 = (bytes[i + 2] as char).to_digit(8); + let o3 = (bytes[i + 3] as char).to_digit(8); + if let (Some(a), Some(b), Some(c)) = (o1, o2, o3) { + let val = (a << 6) | (b << 3) | c; + out.push(val as u8 as char); + i += 4; + continue; + } + } + out.push(bytes[i] as char); + i += 1; + } + out +} + +/// Parse `/proc/mounts`-style content and return mount points whose target is +/// located strictly *under* `path` (i.e. nested inside it, not `path` itself). +/// +/// The list is returned ordered with the deepest paths first, so callers can +/// unmount children before their parents. +fn mounts_under_from_content(content: &str, path: &Path) -> Vec { + let mut mounts: Vec = Vec::new(); + for line in content.lines() { + let mut fields = line.split_whitespace(); + let _device = fields.next(); + let Some(mountpoint) = fields.next() else { + continue; + }; + let unescaped = unescape_mountpath(mountpoint); + let mp = PathBuf::from(&unescaped); + // starts_with is component-based, so /tmp/pkh-1234 is NOT considered + // under /tmp/pkh-123. We additionally exclude the dir itself. + if mp != path && mp.starts_with(path) { + mounts.push(unescaped); + } + } + // Deepest (longest) first so nested mounts are unmounted before parents. + mounts.sort_by_key(|m| std::cmp::Reverse(m.len())); + mounts +} + +/// Discover all prune-able artifacts given the temp dir and cache dir roots. +fn discover_artifacts(temp_dir: &Path, cache_dir: Option<&Path>) -> Vec { + let mut artifacts = Vec::new(); + + if let Ok(entries) = fs::read_dir(temp_dir) { + for entry in entries.flatten() { + let path = entry.path(); + let name = entry.file_name().to_string_lossy().into_owned(); + if !path.is_dir() { + continue; + } + if is_residual_temp_dir(&name) { + artifacts.push(Artifact::TempDir(path)); + } else if name == "pkh-keyrings" { + artifacts.push(Artifact::KeyringDir(path)); + } + } + } + + if let Some(cache) = cache_dir + && let Ok(entries) = fs::read_dir(cache) + { + for entry in entries.flatten() { + let path = entry.path(); + let name = entry.file_name().to_string_lossy().into_owned(); + if name.ends_with(".lock") { + artifacts.push(Artifact::LockFile(path)); + } else if name.ends_with("-buildd.tar.xz") { + artifacts.push(Artifact::Tarball(path)); + } + } + } + + artifacts +} + +/// Remove a path, first attempting a direct removal and only escalating to a +/// privileged `rm -rf` when the direct attempt fails. +/// +/// User-writable artifacts (keyring dir, stale lockfiles) are removed directly +/// without shelling out. Residual build directories may contain root-owned +/// device nodes created via `mknod`, so the direct `remove_dir_all` will fail +/// with `EACCES` and we transparently retry with `sudo rm -rf` (or `rm -rf` when +/// already running as root). +fn remove_path(path: &Path) -> std::io::Result<()> { + let direct = if path.is_dir() { + fs::remove_dir_all(path) + } else { + fs::remove_file(path) + }; + if direct.is_ok() { + return Ok(()); + } + remove_with_privilege(path) +} + +/// Run an `rm -rf ` removal, transparently using `sudo` when the current +/// process does not run as root. This is required because residual build +/// directories may contain root-owned device nodes. +fn remove_with_privilege(path: &Path) -> std::io::Result<()> { + let is_root = crate::utils::root::is_root().unwrap_or(false); + let mut cmd = if is_root { + Command::new("rm") + } else { + let mut c = Command::new("sudo"); + c.arg("rm"); + c + }; + let status = cmd.arg("-rf").arg(path).status()?; + if status.success() { + Ok(()) + } else { + Err(std::io::Error::other(format!( + "rm -rf {} failed with status {}", + path.display(), + status + ))) + } +} + +/// Unmount a single mount point, transparently using `sudo` when not root. +fn unmount(mountpoint: &str) -> std::io::Result<()> { + let is_root = crate::utils::root::is_root().unwrap_or(false); + let mut cmd = if is_root { + Command::new("umount") + } else { + let mut c = Command::new("sudo"); + c.arg("umount"); + c + }; + let status = cmd.arg(mountpoint).status()?; + if status.success() { + Ok(()) + } else { + Err(std::io::Error::other(format!( + "umount {} failed with status {}", + mountpoint, status + ))) + } +} + +/// Execute the prune operation against the given roots, removing residual pkh +/// artifacts. +/// +/// This is the testable core of [`prune()`]: callers pass in the temp +/// directory and optional cache directory to scan. See the +/// [module documentation](self) for what is removed. Returns a [`PruneReport`] +/// describing what was removed (or, in dry-run mode, what would be removed). +pub fn prune_in( + temp_dir: &Path, + cache_dir: Option<&Path>, + options: PruneOptions, +) -> Result> { + let artifacts = discover_artifacts(temp_dir, cache_dir); + + let mut report = PruneReport { + dry_run: options.dry_run, + ..Default::default() + }; + + for artifact in &artifacts { + // Cached tarballs are only removed when --all is requested: they are + // expensive to re-download. + if matches!(artifact, Artifact::Tarball(_)) && !options.all { + continue; + } + + let path = artifact.path(); + log::info!("{}: {}", artifact.kind(), path.display()); + + if options.dry_run { + report.removed.push(path.to_path_buf()); + continue; + } + + // Residual build directories may hold bind-mounted /proc and overlay + // filesystems that must be unmounted before the directory can be + // removed, otherwise `rm -rf` will fail or leave dangling mounts. + if let Artifact::TempDir(_) = artifact { + let mounts_content = fs::read_to_string("/proc/mounts").unwrap_or_default(); + for mp in mounts_under_from_content(&mounts_content, path) { + log::debug!("Unmounting {} under {}", mp, path.display()); + if let Err(e) = unmount(&mp) { + log::warn!("Failed to unmount {}: {}", mp, e); + } + } + } + + match remove_path(path) { + Ok(()) => report.removed.push(path.to_path_buf()), + Err(e) => report.failed.push((path.to_path_buf(), e.to_string())), + } + } + + Ok(report) +} + +/// Execute the prune operation, removing residual pkh artifacts. +/// +/// Convenience wrapper around [`prune_in()`] that uses the system temp +/// directory and the pkh cache directory. See the +/// [module documentation](self) for what is removed. Returns a [`PruneReport`] +/// describing what was removed (or, in dry-run mode, what would be removed). +pub fn prune(options: PruneOptions) -> Result> { + let temp_dir = std::env::temp_dir(); + let cache = cache_dir(); + prune_in(&temp_dir, cache.as_deref(), options) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use tempfile::tempdir; + + #[test] + fn test_is_residual_temp_dir() { + assert!(is_residual_temp_dir("pkh-1700000000")); + assert!(is_residual_temp_dir("pkh-1700000000-1")); + assert!(is_residual_temp_dir("pkh-1700000000-12")); + + // Keyrings and unshare work dirs must NOT match. + assert!(!is_residual_temp_dir("pkh-keyrings")); + assert!(!is_residual_temp_dir("pkh-build-1700000000")); + + // Non-numeric / malformed names. + assert!(!is_residual_temp_dir("pkh-")); + assert!(!is_residual_temp_dir("pkh-abc")); + assert!(!is_residual_temp_dir("pkh-123-abc")); + assert!(!is_residual_temp_dir("pkh-123-")); + assert!(!is_residual_temp_dir("pkh")); + assert!(!is_residual_temp_dir("other-123")); + } + + #[test] + fn test_unescape_mountpath() { + assert_eq!(unescape_mountpath("/tmp/simple"), "/tmp/simple"); + assert_eq!(unescape_mountpath("/tmp/with\\040space"), "/tmp/with space"); + assert_eq!(unescape_mountpath("/tmp/with\\011tab"), "/tmp/with\ttab"); + assert_eq!( + unescape_mountpath("/tmp/with\\134backslash"), + "/tmp/with\\backslash" + ); + // Incomplete escape sequence is left untouched. + assert_eq!(unescape_mountpath("/tmp/with\\04"), "/tmp/with\\04"); + } + + #[test] + fn test_mounts_under_from_content() { + let path = PathBuf::from("/tmp/pkh-100"); + let content = "\ +proc /proc proc rw 0 0 +proc /tmp/pkh-100/proc proc rw,bind 0 0 +overlay /tmp/pkh-100/pkh-overlay/mnt overlay rw 0 0 +overlay /tmp/pkh-1000/pkh-overlay/mnt overlay rw 0 0 +none /tmp/other proc rw 0 0 +"; + let mounts = mounts_under_from_content(content, &path); + // Only mounts strictly under /tmp/pkh-100 should match, deepest first. + assert_eq!(mounts.len(), 2); + assert_eq!(mounts[0], "/tmp/pkh-100/pkh-overlay/mnt"); + assert_eq!(mounts[1], "/tmp/pkh-100/proc"); + } + + #[test] + fn test_discover_artifacts() { + let temp = tempdir().unwrap(); + let cache = tempdir().unwrap(); + let temp_path = temp.path(); + let cache_path = cache.path(); + + // Residual build dirs. + fs::create_dir_all(temp_path.join("pkh-1700000000")).unwrap(); + fs::create_dir_all(temp_path.join("pkh-1700000001-2")).unwrap(); + // Keyring dir. + fs::create_dir_all(temp_path.join("pkh-keyrings")).unwrap(); + // Non-matching entries that should be ignored. + fs::create_dir_all(temp_path.join("pkh-build-1700000000")).unwrap(); + fs::create_dir_all(temp_path.join("other-dir")).unwrap(); + fs::write(temp_path.join("pkh-123-file"), "not a dir").unwrap(); + + // Cache entries. + fs::write(cache_path.join("noble-buildd.tar.xz"), "tarball").unwrap(); + fs::write(cache_path.join("noble-amd64-buildd.tar.xz"), "tarball").unwrap(); + fs::write(cache_path.join("noble-buildd.tar.lock"), "lock").unwrap(); + fs::write(cache_path.join("stray.txt"), "ignore me").unwrap(); + + let artifacts = discover_artifacts(temp_path, Some(cache_path)); + + let mut temp_dirs = 0; + let mut keyring = false; + let mut locks = 0; + let mut tarballs = 0; + for a in &artifacts { + match a { + Artifact::TempDir(_) => temp_dirs += 1, + Artifact::KeyringDir(_) => keyring = true, + Artifact::LockFile(_) => locks += 1, + Artifact::Tarball(_) => tarballs += 1, + } + } + assert_eq!(temp_dirs, 2); + assert!(keyring); + assert_eq!(locks, 1); + assert_eq!(tarballs, 2); + } + + #[test] + fn test_prune_dry_run_keeps_artifacts() { + let temp = tempdir().unwrap(); + let cache = tempdir().unwrap(); + let temp_path = temp.path(); + let cache_path = cache.path(); + + let dir = temp_path.join("pkh-1700000000"); + fs::create_dir_all(&dir).unwrap(); + fs::create_dir_all(temp_path.join("pkh-keyrings")).unwrap(); + fs::write(cache_path.join("noble-buildd.tar.xz"), "tarball").unwrap(); + fs::write(cache_path.join("noble-buildd.tar.lock"), "lock").unwrap(); + + let report = prune_in( + temp_path, + Some(cache_path), + PruneOptions { + dry_run: true, + all: false, + }, + ) + .unwrap(); + + // Dry run reports all discovered artifacts as "removed" without + // touching the filesystem. + assert!(report.dry_run); + assert!(report.removed.iter().any(|p| p == &dir)); + assert!( + report + .removed + .iter() + .any(|p| p == &temp_path.join("pkh-keyrings")) + ); + assert!( + report + .removed + .iter() + .any(|p| p == &cache_path.join("noble-buildd.tar.lock")) + ); + // Tarballs require --all, so not reported here. + assert!( + !report + .removed + .iter() + .any(|p| p == &cache_path.join("noble-buildd.tar.xz")) + ); + + // Nothing was actually removed. + assert!(dir.exists()); + assert!(temp_path.join("pkh-keyrings").exists()); + assert!(cache_path.join("noble-buildd.tar.xz").exists()); + assert!(cache_path.join("noble-buildd.tar.lock").exists()); + } + + #[test] + fn test_prune_removes_user_writable_cache() { + // Removing the keyring dir and stale lockfiles does not require + // privilege escalation since they are user-writable; exercise the real + // prune_in() against controlled temp roots without needing sudo. + let temp = tempdir().unwrap(); + let cache = tempdir().unwrap(); + let temp_path = temp.path(); + let cache_path = cache.path(); + + fs::create_dir_all(temp_path.join("pkh-keyrings")).unwrap(); + fs::write(cache_path.join("noble-buildd.tar.lock"), "lock").unwrap(); + fs::write(cache_path.join("noble-buildd.tar.xz"), "tarball").unwrap(); + + let report = prune_in( + temp_path, + Some(cache_path), + PruneOptions { + dry_run: false, + all: false, + }, + ) + .unwrap(); + + assert!(!report.dry_run); + assert!( + report + .removed + .iter() + .any(|p| p == &temp_path.join("pkh-keyrings")) + ); + assert!( + report + .removed + .iter() + .any(|p| p == &cache_path.join("noble-buildd.tar.lock")) + ); + + assert!(!temp_path.join("pkh-keyrings").exists()); + assert!(!cache_path.join("noble-buildd.tar.lock").exists()); + // Tarball is preserved when --all is not set. + assert!(cache_path.join("noble-buildd.tar.xz").exists()); + } + + #[test] + fn test_prune_all_removes_tarballs() { + let temp = tempdir().unwrap(); + let cache = tempdir().unwrap(); + let cache_path = cache.path(); + + fs::write(cache_path.join("noble-buildd.tar.xz"), "tarball").unwrap(); + fs::write(cache_path.join("noble-amd64-buildd.tar.xz"), "tarball").unwrap(); + fs::write(cache_path.join("noble-buildd.tar.lock"), "lock").unwrap(); + + let report = prune_in( + temp.path(), + Some(cache_path), + PruneOptions { + dry_run: false, + all: true, + }, + ) + .unwrap(); + + assert!( + report + .removed + .iter() + .any(|p| p == &cache_path.join("noble-buildd.tar.xz")) + ); + assert!( + report + .removed + .iter() + .any(|p| p == &cache_path.join("noble-amd64-buildd.tar.xz")) + ); + assert!( + report + .removed + .iter() + .any(|p| p == &cache_path.join("noble-buildd.tar.lock")) + ); + + assert!(!cache_path.join("noble-buildd.tar.xz").exists()); + assert!(!cache_path.join("noble-amd64-buildd.tar.xz").exists()); + assert!(!cache_path.join("noble-buildd.tar.lock").exists()); + } + + #[test] + fn test_prune_removes_user_writable_temp_dirs() { + // Residual build dirs created by the tests are user-writable, so the + // direct fs::remove_dir_all path should succeed without sudo. + let temp = tempdir().unwrap(); + let temp_path = temp.path(); + + fs::create_dir_all(temp_path.join("pkh-1700000000")).unwrap(); + fs::create_dir_all(temp_path.join("pkh-1700000001-2")).unwrap(); + + let report = prune_in(temp_path, None, PruneOptions::default()).unwrap(); + + assert!( + report + .removed + .iter() + .any(|p| p == &temp_path.join("pkh-1700000000")) + ); + assert!( + report + .removed + .iter() + .any(|p| p == &temp_path.join("pkh-1700000001-2")) + ); + + assert!(!temp_path.join("pkh-1700000000").exists()); + assert!(!temp_path.join("pkh-1700000001-2").exists()); + } + + #[test] + fn test_prune_empty_report_when_nothing_found() { + let temp = tempdir().unwrap(); + let cache = tempdir().unwrap(); + // No pkh artifacts at all. + let report = prune_in(temp.path(), Some(cache.path()), PruneOptions::default()).unwrap(); + assert!(report.is_empty()); + } +}