vhaudiquet/pkh
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

deb: rework build sessions into immutable per-build records

Browse Source 
The single mutable session per identity loses the last-good state: a
failed run after a good one adopts the session in place and leaves only
the failed state behind, so "continue from the last success" was
impossible. Each build now records an immutable record under
/var/tmp/pkh/sessions/<slug>/<id>/ (chroot, tree-upper, journal,
manifest), and --incremental forks a NEW record — a full copy of the
base's chroot and, at the same changelog version, its staged artifacts
— under the base's flock, never mutating the base itself.

Selection picks the newest successful record of the identity among the
last five usable, falling back to the newest when none succeeded, and
--incremental <id> pins an exact record (failed ones allowed: their
environment may be fine). The lifecycle is strict keep-iff-incremental:
plain runs build in a throwaway record discarded on success, failure
and interrupt alike (the SIGINT cleanup hook follows the same policy),
while --incremental records are always kept. Each identity keeps its
newest five records; pkh prune's 7-day sweep and orphaned-lock cleanup
follow the nested layout.

--resume and --keep are removed (no aliases): their adoption-in-place
and keep-on-success semantics are exactly what the record model
replaces. pkh deb list shows id, outcome, tree version, fork lineage
and age. README rewritten around the record workflow.
This commit is contained in:
vhaudiquet
2026-09-29 12:56:29 +02:00
parent d65d3264f4
commit 81f0d52f85
6 changed files with 1736 additions and 516 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/deb/mod.rs
+430 -133
View File
@@ -118,30 +118,34 @@ pub struct DebBuildOptions<'a> {
pub jobs: Option<usize>,
/// Explicit build context; defaults to the current context.
pub ctx: Option<Arc<Context>>,
/// Resumable-session request: adopt a previous attempt's session
/// (`pkh deb --resume [<id>]`). Defaults to a fresh build, which
/// replaces the session of this identity without reading it.
pub resume: SessionResume,
/// Keep the session after a successful build (`pkh deb --keep`).
/// Without it a success consumes the session, as today's successful
/// build removes its chroot.
pub keep_session: bool,
/// Incremental request (`pkh deb --incremental [<id>]`): which
/// record to fork as the base of this build, and whether this
/// build's own record is kept. Defaults to a plain throwaway run.
pub incremental: Incremental,
/// Where build events (phases, progress, outcome) are reported.
pub view: &'a dyn BuildView,
}
/// Which session a `pkh deb` build should adopt, if any.
/// The `--incremental` request of a `pkh deb` build: which record the
/// build forks its environment from, and whether its own record is
/// kept afterwards.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub enum SessionResume {
/// No adoption: build from scratch, replacing the session of this
/// identity (recording stays on).
pub enum Incremental {
/// A plain run (`pkh deb` without `--incremental`): build from
/// scratch in a throwaway record, discarded on success, failure
/// and interrupt alike.
#[default]
Fresh,
/// Adopt the newest usable session recorded from this tree
/// (`pkh deb --resume`).
Latest,
/// Adopt the session with this id or unique id prefix
/// (`pkh deb --resume <id>`).
Off,
/// `--incremental`: fork the auto-selected base — the newest
/// successful record of this identity among the last five usable —
/// and always keep the new record. With no usable candidate the
/// build runs from scratch and its record becomes the identity's
/// first base.
Auto,
/// `--incremental <id>`: fork that exact record (unique id or
/// prefix, restricted to the records of this host tree; a FAILED
/// record is a valid base — its environment may be fine, and the
/// user pinned it deliberately) and always keep the new record.
Id(String),
}
@@ -159,8 +163,7 @@ impl Default for DebBuildOptions<'_> {
inject: Vec::new(),
jobs: None,
ctx: None,
resume: SessionResume::Fresh,
keep_session: false,
incremental: Incremental::Off,
view: &QUIET,
}
}
@@ -203,8 +206,7 @@ async fn build_binary_package_impl(
ref inject,
ref jobs,
ref ctx,
ref resume,
keep_session,
ref incremental,
view,
} = opts;
let cwd = cwd.as_deref().unwrap_or_else(|| Path::new("."));
@@ -264,10 +266,11 @@ async fn build_binary_package_impl(
// this scope so it outlives the guarded section below and is only dropped
// once the live view has been cleared.
//
// Resumable build sessions are a local-only feature: with a local base
// context the chroot lives in a session root (recording always on,
// adoption only when requested); every other base context keeps the
// historical anonymous temp chroot.
// Build records are a local-only feature: with a local base context
// every build runs in its own record under the sessions directory
// (forked from a previous one with `--incremental`, thrown away
// otherwise); every other base context keeps the historical anonymous
// temp chroot.
let session_enabled = *mode == BuildMode::Local
&& matches!(base_ctx.config, crate::context::ContextConfig::Local);
let host_tree = cwd.canonicalize().unwrap_or_else(|_| cwd.to_path_buf());
@@ -278,9 +281,20 @@ async fn build_binary_package_impl(
cross,
};
let mut persist_record = false;
let session = if session_enabled {
match resolve_session(resume, &host_tree, &identity, &version, series, chroot_arch) {
Ok(session) => session,
match resolve_record(
incremental,
&host_tree,
&identity,
&version,
series,
chroot_arch,
) {
Ok((record, persist)) => {
persist_record = persist;
Some(record)
}
Err(e) => {
view.finish_failure();
return Err(e);
@@ -289,9 +303,9 @@ async fn build_binary_package_impl(
} else {
None
};
let resume_chroot = session
.as_ref()
.is_some_and(|s| s.chroot_is_usable() && resume != &SessionResume::Fresh);
// Only a forked record carries a bootstrapped chroot at this point;
// fresh ones (plain runs and first `--incremental` runs) extract it.
let resume_chroot = session.as_ref().is_some_and(|s| s.chroot_is_usable());
let mut guard = if let Some(s) = session.clone() {
Some(
@@ -302,7 +316,7 @@ async fn build_binary_package_impl(
base_ctx.clone(),
view,
resume_chroot,
keep_session,
persist_record,
)
.await?,
)
@@ -472,16 +486,19 @@ async fn build_binary_package_impl(
// the chroot) inherit the terminal, so they must not fight the widget.
view.suspend();
// Mark build as successful to trigger chroot cleanup. The guard's drop
// disposes of the session: removed on a plain success, kept with
// `--keep` (outcome recorded there).
// Mark the build as successful for the guard's drop, which disposes of
// the record per the keep policy: an `--incremental` record is kept
// with its outcome recorded, a plain run's throwaway record is
// always discarded.
if result.is_ok()
&& let Some(ref mut g) = guard
&& let Some(g) = &mut guard
{
g.mark_build_successful();
}
// Failures keep the session for a later `--resume`: record the outcome.
// A failed `--incremental` run keeps its record for a later fork:
// record the outcome (discarded plain runs need none).
if result.is_err()
&& persist_record
&& let Some(s) = &session
{
s.set_outcome(session::OUTCOME_FAILED);
@@ -490,133 +507,134 @@ async fn build_binary_package_impl(
result
}
/// Resolve the session a build runs in.
///
/// - [`SessionResume::Fresh`]: open (or create) the session of this
/// identity and reset it for a fresh attempt — the previous attempt's
/// content is replaced without being read. When a concurrent build holds
/// the session, build without one (the historical anonymous temp chroot).
/// - [`SessionResume::Latest`]: adopt the newest usable session recorded
/// from this tree whose identity matches this build; sessions for other
/// identities are a conflict (refused, never silently switched).
/// - [`SessionResume::Id`]: adopt the session matching the id (or unique
/// prefix) recorded from this tree.
///
/// Returns the session handle, if any; `Ok(None, ..)` means "build without
/// a session". The bool says whether the previous attempt's chroot may be
/// reused (it passed the integrity probe and the bootstrap tarball still
/// matches).
fn resolve_session(
resume: &SessionResume,
/// Resolve the record a build runs in, per the `--incremental` request,
/// against the real sessions root and the chroot tarball the bootstrap
/// would use today. See [`resolve_record_in`].
fn resolve_record(
incremental: &Incremental,
host_tree: &Path,
identity: &session::SessionIdentity,
version: &str,
series: &str,
chroot_arch: Option<&str>,
) -> Result<Option<session::Session>, Box<dyn Error>> {
) -> Result<(session::Session, bool), Box<dyn Error>> {
let expected = ephemeral::chroot_tarball_path(series, chroot_arch).ok();
resolve_record_in(
&session::sessions_root(),
expected.as_deref(),
incremental,
host_tree,
identity,
version,
)
}
/// The testable core of [`resolve_record`] over an explicit sessions root
/// and bootstrap tarball.
///
/// Every local build runs in its own record: [`Incremental::Off`] gets a
/// brand-new throwaway one, [`Incremental::Auto`] and
/// [`Incremental::Id`] fork the selected base. The returned flag says
/// whether the record is kept after the build — exactly the
/// `--incremental` runs keep theirs.
///
/// A base must be usable: its chroot passes the integrity probe and was
/// bootstrapped from the tarball this build would use (`expected_tarball`
/// is `None` when the path cannot be resolved — no record is usable
/// then, which degrades an `--incremental` run to a fresh build).
fn resolve_record_in(
records_root: &Path,
expected_tarball: Option<&Path>,
incremental: &Incremental,
host_tree: &Path,
identity: &session::SessionIdentity,
version: &str,
) -> Result<(session::Session, bool), Box<dyn Error>> {
let usable = |s: &session::Session| {
s.chroot_is_usable() && session::tarball_matches(s, series, chroot_arch)
s.chroot_is_usable()
&& expected_tarball.is_some_and(|t| session::tarball_matches_path(s, t))
};
match resume {
SessionResume::Fresh => match session::fresh_session(host_tree, identity, version) {
Ok(s) => Ok(Some(s)),
Err(session::SessionOpenError::Busy(busy)) => {
log::info!("{busy}; building without a session");
Ok(None)
}
Err(session::SessionOpenError::Unusable(e)) => {
log::error!("{e}; building without a session");
Ok(None)
}
Err(session::SessionOpenError::Other(e)) => Err(e.into()),
},
SessionResume::Latest => {
let candidates: Vec<session::Session> = session::for_tree(host_tree)
.into_iter()
.filter(|s| usable(s))
.collect();
match candidates
.iter()
.find(|s| s.manifest().identity == *identity)
{
Some(s) => Ok(Some(adopt_session(s, identity, version)?)),
None if candidates.is_empty() => {
log::info!(
"No resumable build session found for this tree; building from scratch"
);
Ok(None)
}
None => {
let found = candidates
.iter()
.map(|s| s.manifest().identity.to_string())
.collect::<Vec<_>>()
.join(", ");
Err(format!(
"cannot resume: the sessions of this tree were built for {found}, \
but this build targets {identity}. Use `pkh deb list` and \
`pkh deb --resume <id>`, or adjust the selectors"
)
.into())
match incremental {
Incremental::Off => Ok((
session::new_record_in(records_root, host_tree, identity, version)?,
false,
)),
Incremental::Auto => {
// A record locked by a live build cannot be a base: its
// chroot may be mid-mutation (apt always runs, deps land
// inside it), and the fork would copy that state.
let candidates: Vec<session::Session> =
session::identity_records_in(records_root, identity)
.into_iter()
.filter(|s| usable(s))
.filter(|s| !session::lock_is_held(&s.lock_path()).unwrap_or(true))
.take(session::KEEP_RECORDS)
.collect();
for base in session::rank_base_candidates(candidates) {
let why = if base.manifest().last_outcome == session::OUTCOME_SUCCESS {
"the newest successful record of the target"
} else {
"no successful record among the last usable ones; the newest record"
};
match session::fork_record_in(records_root, &base, host_tree, identity, version) {
Ok(record) => {
log::info!(
"Base of the incremental build: record {} ({why})",
base.manifest().id
);
return Ok((record, true));
}
Err(session::SessionOpenError::Busy(busy)) => {
log::info!("{busy}; trying the next candidate record");
}
Err(e) => return Err(e.into()),
}
}
log::info!(
"No usable record of {identity} to fork; building from scratch — \
this run's record becomes the target's first base"
);
Ok((
session::new_record_in(records_root, host_tree, identity, version)?,
true,
))
}
SessionResume::Id(id) => {
let mut matches: Vec<session::Session> = session::by_id_prefix(id)
Incremental::Id(id) => {
let mut matches: Vec<session::Session> = session::by_id_prefix_in(records_root, id)
.into_iter()
.filter(|s| s.manifest().host_tree == host_tree.to_string_lossy())
.filter(|s| usable(s))
.collect();
match matches.len() {
0 => Err(format!(
"no usable build session matches id '{id}' for this tree \
"no usable build record matches id '{id}' for this tree \
(see `pkh deb list`)"
)
.into()),
1 => Ok(Some(adopt_session(&matches.remove(0), identity, version)?)),
1 => {
let base = matches.remove(0);
log::info!(
"Base of the incremental build: record {} (pinned by id)",
base.manifest().id
);
Ok((
session::fork_record_in(records_root, &base, host_tree, identity, version)?,
true,
))
}
_ => {
let found = matches
.iter()
.map(|s| s.manifest().id.clone())
.collect::<Vec<_>>()
.join(", ");
Err(format!("ambiguous session id '{id}': matches {found}").into())
Err(format!("ambiguous record id '{id}': matches {found}").into())
}
}
}
}
}
/// Lock a session for adoption and update its journal: outcome resets to
/// running, a changed changelog version drops the build artifacts (the
/// environment is kept), and the reuse decision for the chroot is made.
fn adopt_session(
session: &session::Session,
identity: &session::SessionIdentity,
version: &str,
) -> Result<session::Session, Box<dyn Error>> {
session
.acquire()
.map_err(|busy| -> Box<dyn Error> { format!("cannot resume: {busy}").into() })?;
let manifest = session.manifest();
if manifest.tree_version != version {
log::info!(
"Version changed since the previous attempt ({}): build artifacts \
are discarded, the environment is reused",
manifest.tree_version
);
session.wipe_build_artifacts();
session.set_tree_version(version);
}
session.set_outcome(session::OUTCOME_RUNNING);
log::info!(
"Resuming session {} for {identity}: the prepared chroot and \
installed build dependencies are reused",
manifest.id
);
Ok(session.clone())
}
/// Resolve the package directory for a build inside the staged build root.
///
/// The tree the caller pointed at is authoritative: `cwd`'s changelog
@@ -1330,4 +1348,283 @@ mod tests {
".deb not produced for the 'tree'-named directory, got: {deb_files:?}"
);
}
// -------------------------------------------------------------------
// Record resolution (`--incremental` base selection)
// -------------------------------------------------------------------
use crate::deb::session::record_fixtures::{
record_with_environment, sessions_fixture, test_identity,
};
use crate::deb::session::{self, SessionIdentity};
/// A plain run resolves to a brand-new, lineage-free record that the
/// caller will discard — including when no record exists at all yet.
#[test]
fn resolve_plain_run_builds_a_fresh_discardable_record() {
let (root, tarball) = sessions_fixture();
let identity = test_identity();
let host_tree = root.path().join("tree");
let (record, persist) = resolve_record_in(
root.path(),
Some(&tarball),
&Incremental::Off,
&host_tree,
&identity,
"1.0",
)
.expect("a plain run always resolves to a fresh record");
assert!(!persist, "a plain run discards its record");
assert_eq!(record.manifest().base, None);
assert!(!record.chroot_is_usable());
}
/// The user scenario, end to end over the record layout: record A
/// succeeds, a fork B of it fails — the default `--incremental`
/// selection forks A again (the newest success), never the failed B.
#[test]
fn resolve_auto_forks_the_newest_success_after_a_failed_fork() {
let (root, tarball) = sessions_fixture();
let identity = test_identity();
let host_tree = root.path().join("tree");
let a = record_with_environment(
root.path(),
&tarball,
&host_tree,
&identity,
"1.0",
Some(session::OUTCOME_SUCCESS),
);
let a_id = a.manifest().id;
let b = session::fork_record_in(root.path(), &a, &host_tree, &identity, "1.0").unwrap();
let b_id = b.manifest().id;
b.set_outcome(session::OUTCOME_FAILED);
drop((a, b));
let (record, persist) = resolve_record_in(
root.path(),
Some(&tarball),
&Incremental::Auto,
&host_tree,
&identity,
"1.0",
)
.expect("the auto selection must fork a usable record");
assert!(persist);
assert_eq!(record.manifest().base.as_deref(), Some(a_id.as_str()));
assert_ne!(record.manifest().id, b_id);
}
/// A first `--incremental` run with no candidate builds completely
/// fresh and persists its record — it becomes the identity's first
/// base.
#[test]
fn resolve_auto_without_candidates_builds_fresh_and_persists() {
let (root, tarball) = sessions_fixture();
let identity = test_identity();
let host_tree = root.path().join("tree");
let (record, persist) = resolve_record_in(
root.path(),
Some(&tarball),
&Incremental::Auto,
&host_tree,
&identity,
"1.0",
)
.expect("no candidates must degrade to a fresh build");
assert!(persist, "the first record becomes a base");
assert_eq!(record.manifest().base, None);
}
/// Non-usable records never make it into the selection: a newer
/// successful record with a broken chroot loses to an older usable
/// failed one.
#[test]
fn resolve_auto_skips_non_usable_records() {
let (root, tarball) = sessions_fixture();
let identity = test_identity();
let host_tree = root.path().join("tree");
let usable = record_with_environment(
root.path(),
&tarball,
&host_tree,
&identity,
"1.0",
Some(session::OUTCOME_FAILED),
);
let usable_id = usable.manifest().id;
drop(usable);
// Newer, successful, but its chroot never finished bootstrapping.
let broken = session::new_record_in(root.path(), &host_tree, &identity, "1.0").unwrap();
broken.set_outcome(session::OUTCOME_SUCCESS);
drop(broken);
let (record, _) = resolve_record_in(
root.path(),
Some(&tarball),
&Incremental::Auto,
&host_tree,
&identity,
"1.0",
)
.expect("the usable record is still a base");
assert_eq!(record.manifest().base.as_deref(), Some(usable_id.as_str()));
}
/// `--incremental <id>` may pin a FAILED record — its environment may
/// be fine and the user chose it deliberately.
#[test]
fn resolve_pinned_id_forks_a_failed_record() {
let (root, tarball) = sessions_fixture();
let identity = test_identity();
let host_tree = root.path().join("tree");
let failed = record_with_environment(
root.path(),
&tarball,
&host_tree,
&identity,
"1.0",
Some(session::OUTCOME_FAILED),
);
let failed_id = failed.manifest().id;
drop(failed);
let (record, persist) = resolve_record_in(
root.path(),
Some(&tarball),
&Incremental::Id(failed_id.clone()),
&host_tree,
&identity,
"1.0",
)
.expect("a pinned failed record is a valid base");
assert!(persist);
assert_eq!(record.manifest().base.as_deref(), Some(failed_id.as_str()));
}
/// A pinned id is restricted to the records of this host tree: an id
/// recorded from another tree is refused.
#[test]
fn resolve_pinned_id_refuses_other_trees() {
let (root, tarball) = sessions_fixture();
let identity = test_identity();
let host_tree = root.path().join("tree");
let other_tree = root.path().join("elsewhere");
let foreign = record_with_environment(
root.path(),
&tarball,
&other_tree,
&identity,
"1.0",
Some(session::OUTCOME_SUCCESS),
);
let foreign_id = foreign.manifest().id;
drop(foreign);
let err = resolve_record_in(
root.path(),
Some(&tarball),
&Incremental::Id(foreign_id),
&host_tree,
&identity,
"1.0",
)
.expect_err("records of other trees must not be forked by id");
assert!(
err.to_string()
.contains("no usable build record matches id"),
"wrong error: {err}"
);
}
/// An ambiguous id prefix is an error naming every match.
#[test]
fn resolve_pinned_id_refuses_ambiguous_prefixes() {
let (root, tarball) = sessions_fixture();
let identity = test_identity();
let host_tree = root.path().join("tree");
drop(record_with_environment(
root.path(),
&tarball,
&host_tree,
&identity,
"1.0",
None,
));
drop(record_with_environment(
root.path(),
&tarball,
&host_tree,
&identity,
"1.0",
None,
));
// All record ids share the date-time prefix of their creation
// second; anything shorter is ambiguous.
let prefix = now_prefix(root.path(), &identity);
let err = resolve_record_in(
root.path(),
Some(&tarball),
&Incremental::Id(prefix),
&host_tree,
&identity,
"1.0",
)
.expect_err("an ambiguous prefix must be refused");
assert!(
err.to_string().contains("ambiguous record id"),
"wrong error: {err}"
);
}
/// The common date-time prefix of every record of the identity (the
/// timestamp of the test-second they were all created in).
fn now_prefix(root: &std::path::Path, identity: &SessionIdentity) -> String {
session::identity_records_in(root, identity)[0]
.manifest()
.id
.chars()
.take(8)
.collect()
}
/// A pinned record of another identity is refused: its environment
/// was bootstrapped for a different target.
#[test]
fn resolve_pinned_id_refuses_foreign_identities() {
let (root, tarball) = sessions_fixture();
let identity = test_identity();
let other = SessionIdentity {
arch: "riscv64".to_string(),
..identity.clone()
};
let host_tree = root.path().join("tree");
let foreign = record_with_environment(
root.path(),
&tarball,
&host_tree,
&other,
"1.0",
Some(session::OUTCOME_SUCCESS),
);
let foreign_id = foreign.manifest().id;
drop(foreign);
let err = resolve_record_in(
root.path(),
Some(&tarball),
&Incremental::Id(foreign_id),
&host_tree,
&identity,
"1.0",
)
.expect_err("a foreign-identity record must not be forked");
assert!(
err.to_string().contains("it was recorded for"),
"wrong error: {err}"
);
}
}