diff --git a/README.md b/README.md index 106a3d1..602e1ed 100644 --- a/README.md +++ b/README.md @@ -184,14 +184,24 @@ pkh deb --no-ccache # build without the shared compile cache ``` Each target keeps at most five staged trees: when a sixth is retained, -the oldest degrades to a metadata-only record (manifest and log stay), -and `pkh prune` removes records older than a week. `pkh prune --dry-run` -previews everything it would remove, grouped by kind with sizes; -`--all` ignores the age and removes everything a live build does not -hold, recent records and staged trees included; `-f/--force` skips the -size pass and the confirmation prompt, removing immediately. `pkh deb ---incremental` never forks a record built for a different -series/architecture. +the oldest degrades to a metadata-only record (manifest and log stay). +A staged tree larger than 20 GiB is never kept at all — a tree that +size costs more disk than the resume it buys, so its record degrades +too and the next build re-stages from scratch. The cap is a setting: + +```json +// ~/.config/pkh/config.json (missing = defaults; keys may be omitted) +{ "staged_maxsize": "20G" } +``` + +`"0"` disables the size rule entirely, and `PKH_STAGED_MAXSIZE` +overrides the file when set. `pkh prune` removes records older than a +week. `pkh prune --dry-run` previews everything it would remove, +grouped by kind with sizes; `--all` ignores the age and removes +everything a live build does not hold, recent records and staged trees +included; `-f/--force` skips the size pass and the confirmation +prompt, removing immediately. `pkh deb --incremental` never forks a +record built for a different series/architecture. Packages whose `debian/rules` keeps an input-untracked stamp cache (the kernels' `debian/stamps`) declare it in the quirks data diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..620ffaf --- /dev/null +++ b/src/config.rs @@ -0,0 +1,102 @@ +//! User settings: the optional `config.json` in the pkh configuration +//! directory (the same `~/.config/pkh/` that holds `contexts.json`), +//! its defaults, and the environment overrides. +//! +//! A missing file carries the defaults — the file is written only when +//! a user creates it, never generated. A file that cannot be read or +//! parsed must not take the whole program down: like the context +//! configuration, it is backed up to `.bak` (best effort) so a +//! later save cannot silently destroy it, and the defaults apply. Every +//! field is individually defaulted (`#[serde(default)]`), so any +//! hand-written file may carry a subset of the settings. + +use serde::Deserialize; +use std::path::PathBuf; + +/// The pkh configuration directory (the one holding `contexts.json` +/// and `config.json`). +pub fn config_dir() -> Option { + directories::ProjectDirs::from("com", "pkh", "pkh").map(|dirs| dirs.config_dir().to_path_buf()) +} + +/// The settings file in [`config_dir`]. +fn config_path() -> Option { + config_dir().map(|dir| dir.join("config.json")) +} + +/// The user-adjustable settings, as stored in `config.json`. New fields +/// must default — the file always may carry a subset. +#[derive(Debug, Clone, Deserialize, Default)] +#[serde(default)] +pub struct Settings { + /// The retention size cap of a staged build tree: trees larger than + /// this are not retained after a build (the record degrades to + /// metadata-only), and existing oversized trees degrade at the next + /// build's retention. A size string (`20G`, `512M`, bytes); `0` + /// disables the cap. + pub staged_maxsize: String, +} + +/// The staged-tree retention cap in bytes: `PKH_STAGED_MAXSIZE` overrides +/// the settings file, which overrides the default (20 GiB — the scale +/// where a carried kernel tree stops earning its disk). +pub fn staged_maxsize() -> u64 { + if let Ok(text) = std::env::var("PKH_STAGED_MAXSIZE") + && let Some(bytes) = crate::utils::size::parse_size(&text) + { + return bytes; + } + file_staged_maxsize() +} + +/// The settings-file layer of [`staged_maxsize`]: the stored value +/// parsed, the default when unset, unparseable, or the file is missing +/// or corrupt (a corrupt file is backed up, as with the context +/// configuration). +fn file_staged_maxsize() -> u64 { + const DEFAULT: u64 = 20 * 1024 * 1024 * 1024; + let Some(path) = config_path() else { + return DEFAULT; + }; + let Ok(text) = std::fs::read_to_string(&path) else { + return DEFAULT; + }; + let settings: Settings = match serde_json::from_str(&text) { + Ok(settings) => settings, + Err(e) => { + log::error!( + "Settings file {} is corrupt ({e}); using the defaults", + path.display() + ); + backup(&path); + return DEFAULT; + } + }; + crate::utils::size::parse_size(&settings.staged_maxsize).unwrap_or_else(|| { + log::warn!( + "Settings file {} carries an unparseable staged_maxsize \ + '{}'; using the default", + path.display(), + settings.staged_maxsize + ); + DEFAULT + }) +} + +/// Back up a corrupt settings file next to itself (best effort), so a +/// later save cannot silently destroy its content. +fn backup(path: &std::path::Path) { + let mut os = path.as_os_str().to_os_string(); + os.push(".bak"); + let backup_path = PathBuf::from(os); + match std::fs::copy(path, &backup_path) { + Ok(_) => log::warn!( + "Corrupt settings file backed up to {}", + backup_path.display() + ), + Err(e) => log::warn!( + "Could not back up corrupt settings file to {}: {e}", + backup_path.display() + ), + } +} diff --git a/src/deb/session.rs b/src/deb/session.rs index 2567a32..996e782 100644 --- a/src/deb/session.rs +++ b/src/deb/session.rs @@ -265,6 +265,11 @@ pub struct SessionManifest { /// before any phase produced a measurement). #[serde(default, skip_serializing_if = "Option::is_none")] pub profile: Option, + /// Size of the retained staged tree in bytes, measured at retention + /// (`None`: never measured — the record predates the field, its tree + /// was kept by an interrupted build, or it retains nothing). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub staged_bytes: Option, } /// State shared through the [`Session`] handle: the manifest plus the @@ -286,12 +291,15 @@ pub struct Session { /// What the end-of-run retention of a record's staged tree does, /// decided from the request and the disk: a keep requested for a run /// that ended before staging created a tree-upper has nothing to keep, -/// and the record must not claim otherwise. +/// and the record must not claim otherwise. A keep of a tree over the +/// retention size cap also drops it: a staged tree only earns its disk +/// by making the next build resume, and a tree too large to keep is too +/// large to keep five of. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum UpperRetention { /// Keep the staged tree: the record is a fork candidate. Keep, - /// Drop the staged tree: a plain run. + /// Drop the staged tree: a plain run, or a tree over the cap. Drop, /// An `--incremental` run whose staging never happened: metadata /// and log only, `kept_upper` stays false. @@ -300,16 +308,42 @@ pub(crate) enum UpperRetention { /// Decide the retention of a record's staged tree: an /// `--incremental` run keeps it — unless the run ended before staging -/// created one, in which case there is nothing to keep — and a plain -/// run drops it. -pub(crate) fn upper_retention(requested: bool, upper_exists: bool) -> UpperRetention { +/// created one (`upper_exists` false), or the tree's measured +/// `upper_bytes` exceed `max_bytes` (0 = unlimited), in which case +/// there is nothing worth keeping — and a plain run drops it. +pub(crate) fn upper_retention( + requested: bool, + upper_exists: bool, + upper_bytes: u64, + max_bytes: u64, +) -> UpperRetention { match (requested, upper_exists) { - (true, true) => UpperRetention::Keep, (true, false) => UpperRetention::KeepNothing, (false, _) => UpperRetention::Drop, + (true, true) if max_bytes != 0 && upper_bytes > max_bytes => UpperRetention::Drop, + (true, true) => UpperRetention::Keep, } } +/// The staged-tree retention cap in bytes (`0` = unlimited), from the +/// user settings (see [`crate::config`]). +pub(crate) fn staged_maxsize() -> u64 { + crate::config::staged_maxsize() +} + +/// The staged tree's size in bytes of a record: the manifest's stored +/// measurement when it has one, a fresh one (recorded for the next +/// sweeps) otherwise — a tree only changes while its own build runs, +/// so a stored value stays accurate between builds. +fn upper_size(record: &Session) -> u64 { + if let Some(bytes) = record.manifest().staged_bytes { + return bytes; + } + let bytes = crate::utils::size::tree_sizes(&[record.upper_dir().as_path()])[0]; + record.set_staged_bytes(bytes); + bytes +} + impl Session { /// The record root directory. pub fn root(&self) -> &Path { @@ -424,6 +458,14 @@ impl Session { }); } + /// Record the measured size of the retained staged tree and persist + /// it (the upper-cap sweeps reuse it instead of re-walking the tree). + pub(crate) fn set_staged_bytes(&self, bytes: u64) { + self.with_manifest(|manifest| { + manifest.staged_bytes = Some(bytes); + }); + } + /// Run `f` over the manifest and persist the result; failures are /// logged and never abort the build (the journal is a cache). fn with_manifest(&self, f: impl FnOnce(&mut SessionManifest)) { @@ -444,46 +486,69 @@ impl Session { /// Retain the record at the end of its build: the outcome is /// recorded on success (failures already carry theirs), and the staged /// tree follows [`upper_retention`] — kept exactly for `--incremental` - /// runs that actually staged one; a plain run's tree is dropped and a - /// run that ended before staging keeps nothing, both leaving a + /// runs that actually staged one under the retention size cap; a + /// plain run's tree is dropped, a run that ended before staging keeps + /// nothing and a tree over the cap is dropped, all leaving a /// metadata-only record (manifest + log). The chroot was already /// removed by the caller's teardown. The interrupt path is handled by /// the SIGINT cleanup hook, which follows the same policy. pub(crate) fn retain_upper(&self, keep_upper: bool, succeeded: bool) { + self.retain_upper_with_cap(keep_upper, succeeded, staged_maxsize()); + } + + /// [`Self::retain_upper`] with the retention cap explicit (the test + /// seam; the cap usually comes from [`staged_maxsize`]). + pub(crate) fn retain_upper_with_cap(&self, keep_upper: bool, succeeded: bool, max_bytes: u64) { if succeeded { self.set_outcome(OUTCOME_SUCCESS); } - match upper_retention(keep_upper, self.has_upper()) { + let id = self.manifest().id; + let upper_bytes = (keep_upper && self.has_upper()) + .then(|| crate::utils::size::tree_sizes(&[self.upper_dir().as_path()])[0]); + match upper_retention( + keep_upper, + self.has_upper(), + upper_bytes.unwrap_or(0), + max_bytes, + ) { UpperRetention::Keep => { self.with_manifest(|manifest| { manifest.kept_upper = true; + manifest.staged_bytes = upper_bytes; }); log::info!( - "Build record {} kept its staged tree (the next \ - --incremental run forks it)", - self.manifest().id + "Build record {id} kept its staged tree ({}, the next \ + --incremental run forks it)", + crate::utils::size::format_bytes(upper_bytes.unwrap_or(0)) ); - self.enforce_upper_cap(); + self.enforce_upper_cap(max_bytes); } UpperRetention::KeepNothing => { log::info!( - "Build record {} kept (no staged tree was created — the \ - run ended before staging)", - self.manifest().id + "Build record {id} kept (no staged tree was created — the \ + run ended before staging)" ); } UpperRetention::Drop => { - log::debug!( - "Dropping the staged tree of build record {} (plain run)", - self.manifest().id - ); + if upper_bytes.is_some() { + log::info!( + "Build record {id} dropped its staged tree ({}, over \ + the retention cap of {}; the next build re-stages \ + from scratch)", + crate::utils::size::format_bytes(upper_bytes.unwrap_or(0)), + crate::utils::size::format_bytes(max_bytes) + ); + } else { + log::debug!("Dropping the staged tree of build record {id} (plain run)"); + } self.wipe_upper(); } } } /// Drop the record's persisted upperdir state: the tree-upper, its - /// host-tree snapshot and the scratch workdir. + /// host-tree snapshot and the scratch workdir, and the manifest's + /// claims about them (`kept_upper`, the measured size). fn wipe_upper(&self) { for path in [self.upper_dir(), self.work_dir(), self.snapshot_file()] { if let Err(e) = crate::deb::ephemeral::privileged_remove(&path) @@ -492,13 +557,22 @@ impl Session { log::warn!("Failed to remove '{}': {}", path.display(), e); } } + self.with_manifest(|manifest| { + manifest.kept_upper = false; + manifest.staged_bytes = None; + }); } - /// Enforce [`KEEP_UPPERS`] for this record's identity: degrade the - /// oldest upper-retaining records beyond the cap to metadata-only - /// (upperdir and snapshot deleted, manifest and log kept). Records - /// locked by a live build are never degraded. - fn enforce_upper_cap(&self) { + /// Enforce the retention caps for this record's identity (the size + /// cap comes from the user settings, [`staged_maxsize`]): degrade to + /// metadata-only (upperdir and snapshot deleted, + /// manifest and log kept) the oldest upper-retaining records beyond + /// [`KEEP_UPPERS`], and every upper-retaining record whose staged + /// tree exceeds `max_bytes` — the newest build's own tree included, + /// covering trees kept oversized by older pkh versions or + /// interrupted builds. Records locked by a live build are never + /// degraded. + fn enforce_upper_cap(&self, max_bytes: u64) { // The record root is //: the sessions // root is two levels up, keeping the cap on the same root the // record lives in (tests use fixture roots). @@ -511,8 +585,15 @@ impl Session { .filter(|record| record.has_upper()) .collect(); uppers.sort_by(|a, b| record_ordering(a, b).reverse()); - for record in uppers.into_iter().skip(KEEP_UPPERS) { - if lock_is_held(&record.lock_path()).unwrap_or(true) { + for (skip, record) in uppers.into_iter().enumerate() { + // The size only decides for records the count cap would keep: + // beyond KEEP_UPPERS the record degrades regardless, and + // measuring its tree would be a wasted walk. + let over_cap = skip < KEEP_UPPERS && max_bytes != 0 && upper_size(&record) > max_bytes; + if skip < KEEP_UPPERS && !over_cap { + continue; + } + if record.root == self.root || lock_is_held(&record.lock_path()).unwrap_or(true) { log::debug!( "Record {} is locked by a live build; skipping the \ upper cap", @@ -520,15 +601,22 @@ impl Session { ); continue; } - log::debug!( - "Record {} degrades to metadata-only (beyond the newest \ - {KEEP_UPPERS} staged trees of {identity})", - record.manifest().id - ); + if over_cap { + log::info!( + "Record {} degrades to metadata-only (its staged tree \ + is {}, over the retention cap of {})", + record.manifest().id, + crate::utils::size::format_bytes(record.manifest().staged_bytes.unwrap_or(0)), + crate::utils::size::format_bytes(max_bytes) + ); + } else { + log::debug!( + "Record {} degrades to metadata-only (beyond the newest \ + {KEEP_UPPERS} staged trees of {identity})", + record.manifest().id + ); + } record.wipe_upper(); - record.with_manifest(|manifest| { - manifest.kept_upper = false; - }); } } @@ -561,7 +649,18 @@ impl Session { /// the interrupted thread). pub(crate) fn record_interrupted(&self, keep_upper: bool) { self.set_outcome(OUTCOME_FAILED); - if upper_retention(keep_upper, self.has_upper()) == UpperRetention::Keep { + let bytes = (keep_upper && self.has_upper()) + .then(|| crate::utils::size::tree_sizes(&[self.upper_dir().as_path()])[0]); + if upper_retention( + keep_upper, + self.has_upper(), + bytes.unwrap_or(0), + staged_maxsize(), + ) == UpperRetention::Keep + { + if let Some(bytes) = bytes { + self.set_staged_bytes(bytes); + } self.with_manifest(|manifest| { manifest.kept_upper = true; }); @@ -993,6 +1092,7 @@ fn fresh_manifest( build_resume: false, timings: None, profile: None, + staged_bytes: None, } } @@ -1214,6 +1314,7 @@ pub(crate) fn fork_record_in( build_resume: false, timings: None, profile: None, + staged_bytes: None, })?; if carry_upper { @@ -2601,6 +2702,7 @@ mod tests { }, ], }), + staged_bytes: None, }; let base = tempfile::tempdir().unwrap(); save_manifest(base.path(), &manifest).unwrap(); @@ -3103,6 +3205,86 @@ mod tests { ); } + /// A staged tree over the retention size cap is not kept: the + /// `--incremental` request cannot keep a tree that costs more disk + /// than it saves, so the record degrades to metadata-only. The cap + /// of 0 (unlimited) keeps the same tree. Sizes are apparent sizes, + /// so the 21 GiB probe file is sparse and costs nothing on disk. + #[test] + fn retain_upper_drops_a_tree_over_the_size_cap() { + let (root, tarball) = sessions_fixture(); + let identity = test_identity(); + let host_tree = root.path().join("tree"); + let over_cap = 21 * 1024 * 1024 * 1024; + let cap = 20 * 1024 * 1024 * 1024; + + let oversized = + record_with_upper(root.path(), &tarball, &host_tree, &identity, "1.0", None); + let file = fs::File::create(oversized.upper_dir().join("big.o")).unwrap(); + file.set_len(over_cap).unwrap(); + drop(file); + + oversized.retain_upper_with_cap(true, true, cap); + assert!( + !oversized.has_upper(), + "a tree over the cap must not be kept" + ); + assert!(!oversized.manifest().kept_upper); + assert!(!oversized.snapshot_file().exists()); + assert_eq!(oversized.manifest().staged_bytes, None); + assert!(oversized.root().join("manifest.json").is_file()); + assert!(oversized.log_file().is_file()); + + // The same tree under an unlimited cap is kept, with its size + // recorded for the cap sweeps. + let kept = record_with_upper(root.path(), &tarball, &host_tree, &identity, "1.1", None); + let file = fs::File::create(kept.upper_dir().join("big.o")).unwrap(); + file.set_len(over_cap).unwrap(); + drop(file); + kept.retain_upper_with_cap(true, true, 0); + assert!(kept.has_upper(), "a cap of 0 keeps any tree"); + assert_eq!( + kept.manifest().staged_bytes, + Some(crate::utils::size::tree_sizes(&[kept.upper_dir().as_path()])[0]), + "the recorded size is the tree's measured size" + ); + } + + /// The retention sweep degrades an older record whose staged tree + /// exceeds the size cap, even though the count cap would keep it: + /// oversized trees left by older pkh versions or interrupted builds + /// come down at the next build's retention. + #[test] + fn the_cap_sweep_degrades_an_oversized_staged_tree() { + let (root, tarball) = sessions_fixture(); + let identity = test_identity(); + let host_tree = root.path().join("tree"); + let cap = 20 * 1024 * 1024 * 1024; + + let old = record_with_upper(root.path(), &tarball, &host_tree, &identity, "1.0", None); + // The stored measurement stands in for a tree the sweep must not + // have to walk: the fixture's own tree is tiny. + old.set_staged_bytes(30 * 1024 * 1024 * 1024); + let old_root = old.root().to_path_buf(); + drop(old); // release the record lock: a held lock is never degraded + + // The new record's tree is tiny: it is kept, and its retention + // sweep takes the old oversized one down. + let new = record_with_upper(root.path(), &tarball, &host_tree, &identity, "1.1", None); + new.retain_upper_with_cap(true, true, cap); + assert!(new.has_upper(), "the new record's tree is under the cap"); + + let degraded = identity_records_in(root.path(), &identity) + .into_iter() + .find(|r| r.root() == old_root) + .expect("the old record is still there"); + assert!(!degraded.has_upper(), "the oversized tree is gone"); + assert!(!degraded.manifest().kept_upper); + assert_eq!(degraded.manifest().staged_bytes, None); + assert!(degraded.root().join("manifest.json").is_file()); + assert!(degraded.log_file().is_file()); + } + /// The end-of-run retention: metadata and the log are kept for every /// build; the staged tree exactly for `--incremental` ones. #[test] @@ -3156,10 +3338,14 @@ mod tests { fn upper_retention_reflects_the_disk_not_the_request() { use UpperRetention::{Drop, Keep, KeepNothing}; - assert_eq!(upper_retention(true, true), Keep); - assert_eq!(upper_retention(true, false), KeepNothing); - assert_eq!(upper_retention(false, true), Drop); - assert_eq!(upper_retention(false, false), Drop); + // An unlimited cap (0) and a generous one keep a tree under them. + assert_eq!(upper_retention(true, true, 5, 0), Keep); + assert_eq!(upper_retention(true, true, 5, 10), Keep); + // Over the cap the tree is dropped, not kept half-claimed. + assert_eq!(upper_retention(true, true, 11, 10), Drop); + assert_eq!(upper_retention(true, false, 5, 10), KeepNothing); + assert_eq!(upper_retention(false, true, 5, 10), Drop); + assert_eq!(upper_retention(false, false, 5, 10), Drop); } /// Driving the keep path on an incremental record whose staged tree diff --git a/src/lib.rs b/src/lib.rs index a1d7f4b..7f1cce1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,9 @@ pub mod apt; pub mod build; /// Parse or edit a Debian changelog of a source package pub mod changelog; +/// User settings: the optional `config.json` in the pkh configuration +/// directory, its defaults and the environment overrides +pub mod config; /// Embedding convention for static reference data (`data/*.yml`), applied /// by each owning module via the `embed_data!` macro pub(crate) mod data; diff --git a/src/utils/size.rs b/src/utils/size.rs index a8b290b..c84223e 100644 --- a/src/utils/size.rs +++ b/src/utils/size.rs @@ -111,6 +111,35 @@ pub(crate) fn format_bytes(bytes: u64) -> String { } } +/// Parse a human size into bytes: an integer with an optional unit +/// suffix — `B`, `K`, `M`, `G`, `T` (optionally `iB`, case-insensitive), +/// the binary multiples ccache's configuration uses (`20G` = 20 GiB). A +/// bare number is bytes. `None` on anything else (empty, negative, +/// overflow, unknown unit). +pub(crate) fn parse_size(text: &str) -> Option { + let text = text.trim(); + let split = text + .rfind(|c: char| c.is_ascii_digit()) + .map(|idx| idx + 1) + .unwrap_or(0); + let (digits, unit) = text.split_at(split); + let power = match unit.trim().to_ascii_lowercase().as_str() { + "" | "b" => 0, + "k" | "kib" => 1, + "m" | "mib" => 2, + "g" | "gib" => 3, + "t" | "tib" => 4, + _ => return None, + }; + if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + digits + .parse::() + .ok() + .and_then(|value| value.checked_mul(1u64.checked_shl(10 * power)?)) +} + #[cfg(test)] mod tests { use super::*; @@ -159,4 +188,32 @@ mod tests { // this size; the mount-crossing walk keeps it out of the totals. assert_eq!(format_bytes(140_737_477_959_680), "128.0 TiB"); } + + /// Unit spellings and their byte values: ccache-style binary + /// multiples, optional B/iB decorations, case-insensitive. + #[test] + fn parse_size_reads_the_unit_suffixes() { + assert_eq!(parse_size("0"), Some(0)); + assert_eq!(parse_size("512"), Some(512)); + assert_eq!(parse_size("512B"), Some(512)); + assert_eq!(parse_size("512 b"), Some(512)); + assert_eq!(parse_size("10K"), Some(10 * 1024)); + assert_eq!(parse_size("512m"), Some(512 * 1024 * 1024)); + assert_eq!(parse_size("20G"), Some(20 * 1024 * 1024 * 1024)); + assert_eq!(parse_size("2TiB"), Some(2 * 1024 * 1024 * 1024 * 1024)); + assert_eq!(parse_size(" 20G "), Some(20 * 1024 * 1024 * 1024)); + } + + /// Garbage is an error, not a guess: empty, negative, junk units and + /// overflow all parse to None. + #[test] + fn parse_size_rejects_garbage() { + assert_eq!(parse_size(""), None); + assert_eq!(parse_size(" "), None); + assert_eq!(parse_size("G"), None); + assert_eq!(parse_size("-1"), None); + assert_eq!(parse_size("20GB"), None); + assert_eq!(parse_size("20 kg"), None); + assert_eq!(parse_size("999999999999999999999999T"), None); + } }