From 299b5c28c36bf41c536c509fdfba820786afaaa2 Mon Sep 17 00:00:00 2001 From: Valentin Haudiquet Date: Sun, 27 Sep 2026 21:00:34 +0200 Subject: [PATCH] test: add pkh test, the native DEP-8 runner for local and chroot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DEP-8 run-time contract is one page, so pkh implements it natively instead of wrapping autopkgtest (see plans/pkh-test.md): the testbed is a pkh context — chroot mode reuses the deb ephemeral chroots and their cached mmdebstrap tarballs, local mode runs on the current context — and the binaries under test are the freshness-guarded pkh deb output next to the tree, rebuilt through the deb flow when stale or missing. Per test: restrictions gate on the testbed (isolation-machine, needs-reboot and container isolation skip with a pointer at --mode vm), the stanza Depends are expanded (@, @builddeps@, @recommends@) and installed, then the script or Test-Command runs in the staged tree with the AUTOPKGTEST_* (and legacy ADT_*) environment, a per-test timeout and the stderr/exit-77 verdict rules; flaky tests retry once. stdout, stderr and $AUTOPKGTEST_ARTIFACTS are collected per test under /var/tmp/pkh/tests/ (pruned separately). --shell/--shell-fail drop into the testbed for debugging. pin_pocket and install_injected_packages become crate-internal helpers shared with the deb flow, which they mirror. --- src/deb/local.rs | 4 +- src/deb/mod.rs | 2 +- src/lib.rs | 2 + src/test/bed.rs | 88 +++++++ src/test/expand.rs | 248 +++++++++++++++++ src/test/mod.rs | 628 ++++++++++++++++++++++++++++++++++++++++++++ src/test/options.rs | 204 ++++++++++++++ src/test/outcome.rs | 332 +++++++++++++++++++++++ src/test/runner.rs | 544 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 2049 insertions(+), 3 deletions(-) create mode 100644 src/test/bed.rs create mode 100644 src/test/expand.rs create mode 100644 src/test/mod.rs create mode 100644 src/test/options.rs create mode 100644 src/test/outcome.rs create mode 100644 src/test/runner.rs diff --git a/src/deb/local.rs b/src/deb/local.rs index 3579672..9ca3216 100644 --- a/src/deb/local.rs +++ b/src/deb/local.rs @@ -1075,7 +1075,7 @@ fn options_declare_single_debian_patch(local_options: Option<&str>, options: Opt /// suite on every repository carrying it (archive, security and ports), /// for all architectures, so this also covers cross-builds pulling /// dependencies from 'ports.ubuntu.com'. -fn pin_pocket(pocket_suite: &str, ctx: &Arc) -> Result<(), Box> { +pub(crate) fn pin_pocket(pocket_suite: &str, ctx: &Arc) -> Result<(), Box> { let pin_path = format!("/etc/apt/preferences.d/pkh-{}", pocket_suite); let pin_content = format!( "Package: *\nPin: release a={}\nPin-Priority: 600\n", @@ -1086,7 +1086,7 @@ fn pin_pocket(pocket_suite: &str, ctx: &Arc) -> Result<(), Box, ctx: Arc, diff --git a/src/deb/mod.rs b/src/deb/mod.rs index 78b7d8b..05b2d18 100644 --- a/src/deb/mod.rs +++ b/src/deb/mod.rs @@ -2,7 +2,7 @@ mod cross; /// Ephemeral (per-build) unshare contexts, including the process-global /// cleanup-hook registry drained by the SIGINT handler pub(crate) mod ephemeral; -mod local; +pub(crate) mod local; /// Resumable build sessions: manifest, discovery, `pkh deb list` and the /// host-tree snapshot/sync that keeps build artifacts alive across attempts pub mod session; diff --git a/src/lib.rs b/src/lib.rs index e89b99c..1c8854b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,6 +38,8 @@ pub mod pull; pub mod put; /// Handle package-specific quirks and workarounds pub mod quirks; +/// Run a source package's DEP-8 as-installed tests (`pkh test`) +pub mod test; /// Line classifiers rewriting raw subprocess output into display actions /// and countable progress (pure logic, shared by build views) diff --git a/src/test/bed.rs b/src/test/bed.rs new file mode 100644 index 0000000..8130550 --- /dev/null +++ b/src/test/bed.rs @@ -0,0 +1,88 @@ +//! Testbed lifecycle: where and how the tests run. +//! +//! A testbed is a [`Context`] plus its teardown. chroot mode reuses the +//! `pkh deb` machinery wholesale (an ephemeral unshare context guarded by +//! [`crate::deb::ephemeral::EphemeralContextGuard`], bootstrapped from the +//! same cached mmdebstrap tarballs); local mode is the current context +//! used directly. The testbed itself is disposable per run: autopkgtests +//! may break it, and the next run starts from a clean bootstrap. + +use std::error::Error; +use std::sync::Arc; + +use crate::context::Context; +use crate::deb::ephemeral::EphemeralContextGuard; +use crate::report::BuildView; +use crate::test::options::TestMode; + +/// A prepared testbed, torn down on drop. +pub(crate) enum Bed { + /// An ephemeral unshare chroot. + Chroot(EphemeralContextGuard), + /// The current context, used directly. + Local(Arc), +} + +impl Bed { + /// Prepare a testbed of `mode` for `series`/`arch` on `base_ctx`. + pub(crate) async fn prepare( + mode: TestMode, + series: &str, + arch: &str, + base_ctx: Arc, + view: &dyn BuildView, + ) -> Result> { + match mode { + TestMode::Chroot => { + // Mirror the deb flow: only a foreign target requests a + // foreign-arch chroot, so the native case shares deb's + // unsuffixed `-buildd.tar.xz` cache entry. + let chroot_arch = (arch != crate::get_current_arch()).then(|| arch.to_string()); + let guard = EphemeralContextGuard::new_with_context( + series, + chroot_arch.as_deref(), + base_ctx, + view, + ) + .await?; + Ok(Bed::Chroot(guard)) + } + TestMode::Local => Ok(Bed::Local(base_ctx)), + } + } + + /// The context to run testbed commands in. + pub(crate) fn ctx(&self) -> Arc { + match self { + Bed::Chroot(guard) => guard.context(), + Bed::Local(ctx) => Arc::clone(ctx), + } + } + + /// A scratch root inside the testbed for the staged tree, test + /// temporary directories and artifacts. + pub(crate) fn scratch_root(&self) -> Result> { + match self { + // A stable root inside the chroot; the guard creates a fresh + // temp directory for anonymous chroots. + Bed::Chroot(guard) => guard.build_root(), + Bed::Local(ctx) => Ok(ctx.create_temp_dir()?), + } + } + + /// Whether root-requiring tests can run: always inside a chroot + /// (commands run as mapped root), on the local context only when the + /// context itself runs as root. + pub(crate) fn supports_root(&self) -> bool { + match self { + Bed::Chroot(_) => true, + Bed::Local(ctx) => matches!( + ctx.command("id") + .arg("-u") + .output() + .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_string()), + Ok(id) if id == "0" + ), + } + } +} diff --git a/src/test/expand.rs b/src/test/expand.rs new file mode 100644 index 0000000..7641c6e --- /dev/null +++ b/src/test/expand.rs @@ -0,0 +1,248 @@ +//! DEP-8 `Depends:` substitution: `@`, `@builddeps@` and `@recommends@` +//! expanded into concrete package specs for `apt-get install`. +//! +//! The built binaries themselves are installed separately (from the +//! staged `.deb` files, so tests exercise exactly what pkh built); +//! expansion substitutes bare binary *names*, which apt then resolves +//! against the installed binaries first and the archive second. + +use crate::debian::control::ControlInfo; +use crate::debian::deps::{Deps, ParseOpts}; + +/// Expand a test stanza's `Depends:` value into apt package specs. +/// +/// `raw` is the value as written (`None` = absent, the DEP-8 default +/// `@`). The substitutions, per DEP-8: +/// +/// - `@` in an or-clause replicates the clause once per binary package +/// built from the source, with the name substituted (other +/// alternatives and version constraints are kept); +/// - `@builddeps@` becomes the source's Build-Depends, +/// Build-Depends-Indep and Build-Depends-Arch plus `build-essential`; +/// - `@recommends@` becomes the Recommends of every binary package, +/// with `${...}` substvar references dropped (autopkgtest's +/// "variables stripped") but versions kept. +/// +/// Returns one apt spec per comma-separated clause, in order; an empty +/// result means nothing to install. +pub fn expand_depends( + raw: Option<&str>, + control: &ControlInfo, + binaries: &[String], + host_arch: &str, +) -> Result, String> { + let raw = raw.unwrap_or("@"); + let mut specs = Vec::new(); + for token in split_top_level_commas(raw) { + match token.as_str() { + "@builddeps@" => { + for field in ["Build-Depends", "Build-Depends-Indep", "Build-Depends-Arch"] { + if let Some(value) = non_empty(control.source.get(field)) { + for clause in split_top_level_commas(value) { + if !clause.is_empty() { + specs.push(clause); + } + } + } + } + specs.push("build-essential".to_string()); + } + "@recommends@" => { + for paragraph in &control.binaries { + if let Some(value) = non_empty(paragraph.get("Recommends")) { + specs.extend(name_only_clauses(value, host_arch)?); + } + } + } + token if token.split_whitespace().any(|word| word == "@") => { + // Textual substitution: DEP-8 @ is a word-level + // placeholder the deps grammar does not know. The clause + // replicates once per binary with the name substituted, + // constraints and alternatives kept verbatim. + for binary in binaries { + specs.push( + token + .split_whitespace() + .map(|word| { + if word == "@" { + binary.clone() + } else { + word.to_string() + } + }) + .collect::>() + .join(" "), + ); + } + } + token => specs.push(token.to_string()), + } + } + Ok(specs) +} + +/// Split on commas that sit outside any parentheses: or-clauses carry +/// their own commas inside `(...)` constraints, top-level commas separate +/// dependencies. +fn split_top_level_commas(input: &str) -> Vec { + let mut parts = Vec::new(); + let mut depth = 0usize; + let mut current = String::new(); + for character in input.chars() { + match character { + '(' => { + depth += 1; + current.push(character); + } + ')' => { + depth = depth.saturating_sub(1); + current.push(character); + } + ',' if depth == 0 => { + parts.push(current.trim().to_string()); + current.clear(); + } + _ => current.push(character), + } + } + parts.push(current.trim().to_string()); + parts +} + +/// Render each or-clause of a dependency list, dropping alternatives +/// that are `${...}` substvar references (autopkgtest's "variables +/// stripped") and keeping everything else verbatim. +fn name_only_clauses(input: &str, host_arch: &str) -> Result, String> { + let opts = ParseOpts::build_deps(host_arch.to_string(), Vec::new()); + let deps = Deps::parse(input, &opts)?; + let mut clauses = Vec::new(); + for clause in deps.clauses() { + let kept: Vec = clause + .iter() + .filter(|relation| !relation.package.contains('$')) + .map(|relation| relation.output()) + .collect(); + if !kept.is_empty() { + clauses.push(kept.join(" | ")); + } + } + Ok(clauses) +} + +/// A field value with its surrounding whitespace trimmed; `None` when the +/// field is absent or whitespace-only. +fn non_empty(value: Option<&str>) -> Option<&str> { + value.map(str::trim).filter(|value| !value.is_empty()) +} + +/// The binary package names declared by a control file, in stanza order. +pub fn binary_names(control: &ControlInfo) -> Vec { + control + .binaries + .iter() + .filter_map(|paragraph| paragraph.get("Package")) + .map(|name| name.trim().to_string()) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::str::FromStr; + + fn control() -> ControlInfo { + ControlInfo::from_str( + "Source: hello\n\ + Build-Depends: debhelper-compat (= 13),\n meson\n\ + Build-Depends-Indep: dh-python\n\ + \n\ + Package: hello\n\ + Architecture: any\n\ + Recommends: hello-doc, pulseaudio (>= 15)\n\ + \n\ + Package: hello-doc\n\ + Architecture: all\n\ + Recommends: hello\n", + ) + .unwrap() + } + + fn binaries() -> Vec { + vec!["hello".to_string(), "hello-doc".to_string()] + } + + /// The default (absent) Depends is `@`: one spec per binary. + #[test] + fn default_is_at() { + let specs = expand_depends(None, &control(), &binaries(), "amd64").unwrap(); + assert_eq!(specs, ["hello", "hello-doc"]); + } + + /// An or-clause containing @ replicates once per binary, keeping the + /// other alternatives and version constraints. + #[test] + fn or_clauses_replicate_per_binary() { + let specs = expand_depends( + Some("@ (>= 1.0) | fallback, jq"), + &control(), + &binaries(), + "amd64", + ) + .unwrap(); + assert_eq!( + specs, + [ + "hello (>= 1.0) | fallback", + "hello-doc (>= 1.0) | fallback", + "jq" + ] + ); + } + + /// @builddeps@ pulls the three Build-* fields plus build-essential. + #[test] + fn builddeps_expansion() { + let specs = + expand_depends(Some("@builddeps@, @"), &control(), &binaries(), "amd64").unwrap(); + assert_eq!( + specs, + [ + "debhelper-compat (= 13)", + "meson", + "dh-python", + "build-essential", + "hello", + "hello-doc" + ] + ); + } + + /// @recommends@ collects every binary's Recommends, stripped to names. + #[test] + fn recommends_expansion() { + let specs = expand_depends(Some("@recommends@"), &control(), &binaries(), "amd64").unwrap(); + assert_eq!(specs, ["hello-doc", "pulseaudio (>= 15)", "hello"]); + } + + /// Plain dependencies pass through untouched; a token mentioning @ in + /// a version constraint is still a substitution. + #[test] + fn plain_tokens_pass_through() { + let specs = expand_depends( + Some("ca-certificates, python3:any"), + &control(), + &binaries(), + "amd64", + ) + .unwrap(); + assert_eq!(specs, ["ca-certificates", "python3:any"]); + } + + /// Commas inside version constraints do not split clauses. + #[test] + fn parenthesised_commas_stay_together() { + let specs = + expand_depends(Some("foo (>= 1) , bar"), &control(), &binaries(), "amd64").unwrap(); + assert_eq!(specs, ["foo (>= 1)", "bar"]); + } +} diff --git a/src/test/mod.rs b/src/test/mod.rs new file mode 100644 index 0000000..0d0c58c --- /dev/null +++ b/src/test/mod.rs @@ -0,0 +1,628 @@ +//! Run a source package's DEP-8 as-installed tests (`pkh test`). +//! +//! The runner is native for the chroot and local testbeds: the testbed +//! is a pkh context (an ephemeral unshare chroot bootstrapped exactly +//! like a `pkh deb` build chroot, or the current context directly), and +//! the DEP-8 per-test contract lives in [`runner`]. `--mode vm` execs +//! the installed autopkgtest with its qemu runner instead — the one +//! backend pkh does not reimplement (serial-console protocol, image +//! management, reboot support). +//! +//! Binaries under test are the fresh `pkh deb` output next to the tree +//! (freshness-guarded like the lint wrapper); when they are missing or +//! stale, the deb flow builds them in-process first. + +pub(crate) mod bed; +pub mod expand; +pub mod options; +pub mod outcome; +pub(crate) mod runner; + +pub use options::{TestMode, TestOptions}; +pub use outcome::TestRunReport; + +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use crate::context::{LineSink, Stream}; +use crate::debian::control::{ControlInfo, parse_paragraphs}; +use crate::debian::tests_control::TestsControl; +use crate::logfmt::GenericClassifier; +use crate::report::{BuildTarget, BuildView, TargetKind}; +use crate::test::bed::Bed; +use crate::test::outcome::Verdict; + +/// Where the built binaries under test came from. +struct Binaries { + /// Host-side paths of the .deb files. + paths: Vec, + /// Provenance note for the report. + note: String, +} + +/// Run the DEP-8 tests of a source package tree. +/// +/// Errors are runtime errors (no declared tests, unusable testbed, +/// failed in-process build); test failures are a report, not an error. +pub async fn run(opts: TestOptions<'_>) -> Result> { + let view = opts.view; + let tree = opts + .tree + .clone() + .unwrap_or_else(|| std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."))); + let tree = tree + .canonicalize() + .map_err(|e| format!("cannot access the source tree '{}': {e}", tree.display()))?; + + // Package identity and target, resolved like a deb build. + let entry = crate::debian::parse_changelog_entry(&tree.join("debian/changelog")) + .map_err(|e| format!("cannot read the package identity: {e}"))?; + let package = entry.source.clone(); + let version = entry.version.full(); + let dist = opts + .dist + .clone() + .unwrap_or_else(crate::build::env::current_vendor); + let series = match opts.series.clone() { + Some(series) => series, + None if crate::distro_info::is_unreleased(&entry.distribution) => { + let resolved = crate::distro_info::effective_series(&entry.distribution, &dist).await?; + log::info!( + "Changelog is UNRELEASED, testing against series {}", + resolved + ); + resolved + } + None => entry.distribution.clone(), + }; + let arch = opts.arch.clone().unwrap_or_else(crate::get_current_arch); + + // Pre-check: the package must declare tests, before any testbed is + // booted. + let control_path = tree.join("debian/tests/control"); + if !control_path.exists() { + return Err( + "the package declares no autopkgtests (no debian/tests/control). \ + Add debian/tests/control and 'Testsuite: autopkgtest' to debian/control \ + (pkh lint flags this too), or point pkh test at another tree." + .into(), + ); + } + let tests_control = TestsControl::parse(&control_path) + .map_err(|e| format!("cannot parse debian/tests/control: {e}"))?; + if tests_control.tests.is_empty() { + return Err("debian/tests/control declares no tests".into()); + } + + // Binaries under test: fresh pkh deb output, an in-process build, or + // an explicit --debs list. + let binaries = resolve_binaries(&opts, &tree, &package, &version, &arch, view).await?; + + view.target(BuildTarget { + package: &package, + version: &version, + target: &format!("{series}/{arch}"), + display: format!( + "Testing {} ({}) on {}/{} ({})", + package, + version, + series, + arch, + opts.mode.unwrap_or_default().label() + ), + kind: TargetKind::Test, + tee_log: true, + }); + + let mode = opts.mode.unwrap_or_default(); + if mode == TestMode::Local { + log::warn!( + "Tests run directly on the current context: they install packages \ + and can rewrite it. Use the default chroot mode for isolation." + ); + } + let run_dir = create_run_dir(&package, &series, &arch)?; + let sink = view.sink().or_else(|| { + // Verbose runs without a live view still stream test output live. + opts.verbose + .then(|| Arc::new(TerminalLines) as Arc) + }); + + let base_ctx = crate::context::current(); + let bed = Bed::prepare(mode, &series, &arch, base_ctx, view).await?; + let ctx = bed.ctx(); + + let mut env = HashMap::new(); + env.insert("LANG".to_string(), "C".to_string()); + env.insert("DEBIAN_FRONTEND".to_string(), "noninteractive".to_string()); + if matches!(bed, Bed::Chroot(_)) { + env.insert("HOME".to_string(), "/root".to_string()); + env.insert("USER".to_string(), "root".to_string()); + env.insert("LOGNAME".to_string(), "root".to_string()); + } + + configure_testbed(&ctx, &env, &series, &arch, &opts, view).await?; + + // Stage the tree and the binaries into the testbed. The copy also + // protects the host tree: tests may write into it (rw-build-tree). + view.message("Staging the tree and binaries"); + let staging_root = bed.scratch_root()?; + let staged_tree = ctx.ensure_available(&tree, &staging_root)?; + let staged_tree = staged_tree.to_string_lossy().into_owned(); + let mut staged_debs = Vec::new(); + for deb in &binaries.paths { + staged_debs.push(ctx.ensure_available(deb, &staging_root)?); + } + + // Install the exact binaries under test before the tests run. + if !staged_debs.is_empty() { + view.phase( + "Installing built binaries", + Box::new(GenericClassifier::new()), + ); + let mut cmd = ctx.command("apt-get"); + cmd.envs(env.clone()) + .arg("-y") + .arg("--allow-downgrades") + .arg("install"); + for deb in &staged_debs { + cmd.arg(format!( + "./{}", + deb.to_string_lossy().trim_start_matches('/') + )); + } + let status = cmd.status().map_err(|e| e.to_string())?; + if !status.success() { + return Err("installing the built binaries into the testbed failed".into()); + } + } + + // What @ expands to: the names of the binaries actually under test, + // from their .deb file names (__.deb). + let control_info = ControlInfo::parse(&tree.join("debian/control")) + .map_err(|e| format!("cannot parse debian/control: {e}"))?; + let binary_names: Vec = staged_debs + .iter() + .filter_map(|deb| deb_binary_name(deb)) + .collect(); + let run_ctx = runner::RunContext { + ctx, + tree: staged_tree.clone(), + staging_root, + arch: arch.clone(), + host_arch: crate::get_current_arch(), + supports_root: bed.supports_root(), + timeout_secs: opts.timeout_secs(), + sink, + view, + control: control_info, + binaries: binary_names, + env: env.clone(), + artifacts_dir: run_dir.clone(), + }; + let outcomes = runner::run_tests( + &run_ctx, + &tests_control.tests, + &opts.test_names, + &opts.skip_tests, + ); + + // Debugging escape hatches. + let has_failures = outcomes + .iter() + .any(|outcome| outcome.verdict == Verdict::Fail); + if opts.shell || (opts.shell_fail && has_failures) { + drop_shell(&bed, &staged_tree, &env, view)?; + } + + let report = TestRunReport { + package, + version, + series, + arch, + mode: mode.label().to_string(), + binaries: binaries.paths, + binaries_note: binaries.note, + outcomes, + artifacts_dir: run_dir.clone(), + raw_exit_code: None, + }; + if report.has_failures() { + view.finish_failure(); + } else { + view.finish_success(&[run_dir]); + } + Ok(report) +} + +/// Resolve the binaries under test: `--debs` overrides everything; +/// otherwise the fresh `pkh deb` output next to the tree (freshness- +/// guarded, the lint wrapper's artifact strategy); otherwise the deb +/// flow builds them in-process. `--no-build` refuses instead. +async fn resolve_binaries( + opts: &TestOptions<'_>, + tree: &Path, + package: &str, + version: &str, + arch: &str, + view: &dyn BuildView, +) -> Result> { + if !opts.debs.is_empty() { + for deb in &opts.debs { + if !deb.exists() { + return Err(format!("--deb file '{}' does not exist", deb.display()).into()); + } + } + return Ok(Binaries { + paths: opts.debs.clone(), + note: "explicitly passed with --debs".to_string(), + }); + } + + if !opts.build + && let Some(paths) = fresh_deb_output(tree, package, version, arch) + { + log::info!("Testing the fresh pkh deb output next to the tree"); + return Ok(Binaries { + paths, + note: "fresh pkh deb output".to_string(), + }); + } + + if opts.no_build { + return Err( + "no fresh pkh deb output next to the tree and --no-build given: \ + run 'pkh deb' first, drop --no-build, or pass --debs" + .into(), + ); + } + + view.message("No fresh pkh deb output: building the binaries first"); + let artifacts = crate::deb::build_binary_package(crate::deb::DebBuildOptions { + series: opts.series.clone(), + arch: opts.arch.clone(), + pocket: opts.pocket.clone(), + ppa: opts.ppa.clone(), + cwd: Some(tree.to_path_buf()), + view, + ..Default::default() + }) + .await?; + let debs: Vec = artifacts + .into_iter() + .filter(|path| path.extension().is_some_and(|ext| ext == "deb")) + .collect(); + if debs.is_empty() { + return Err("the build produced no binary packages to test".into()); + } + Ok(Binaries { + paths: debs, + note: "built by pkh deb just now".to_string(), + }) +} + +/// The fresh `pkh deb` output next to the tree, when it matches the +/// current changelog entry and no tree content is newer than it: the +/// `__.changes` file gates, its checksum lists +/// name the debs. Stale or missing output tests nothing. +fn fresh_deb_output(tree: &Path, package: &str, version: &str, arch: &str) -> Option> { + let entry = crate::debian::parse_changelog_entry(&tree.join("debian/changelog")).ok()?; + let parent = tree.parent()?; + let changes_path = parent.join(format!( + "{}_{}_{}.changes", + package, + entry.version.no_epoch(), + arch + )); + let changes = fs::read_to_string(&changes_path).ok()?; + + // Only the current entry's binaries are the ones under test. + let paragraph = parse_paragraphs(&changes).into_iter().next()?; + let matches = paragraph.get("Source").map(str::trim) == Some(package) + && paragraph.get("Version").map(str::trim) == Some(version); + if !matches { + return None; + } + + let mut debs = Vec::new(); + for (field, value) in paragraph.iter() { + if !field.starts_with("Checksums-") && field != "Files" { + continue; + } + for line in value.lines() { + // `
[] ` + if let Some(name) = line.split_whitespace().last() + && name.ends_with(".deb") + { + debs.push(parent.join(name)); + } + } + } + if debs.is_empty() || debs.iter().any(|deb| !deb.exists()) { + return None; + } + // A tree file written after the changes file makes the output stale. + let built = fs::metadata(&changes_path).ok()?.modified().ok()?; + if tree_newer_than(tree, built) { + return None; + } + Some(debs) +} + +/// Whether any tree content is newer than `built`. Skips `.git` and +/// `.pc`: commits and quilt bookkeeping churn their mtimes without +/// touching what the package contains. +fn tree_newer_than(root: &Path, built: std::time::SystemTime) -> bool { + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + let name = entry.file_name(); + if name == ".git" || name == ".pc" { + continue; + } + let Ok(meta) = fs::symlink_metadata(&path) else { + continue; + }; + if meta.is_dir() { + stack.push(path); + } else if meta.modified().is_ok_and(|mtime| mtime > built) { + return true; + } + } + } + false +} + +/// Configure the testbed's APT setup and update it: pockets and PPAs on +/// the archive sources (the same rules as the deb build), an optional +/// upgrade, the caller's setup commands and injected packages. +async fn configure_testbed( + ctx: &Arc, + env: &HashMap, + series: &str, + arch: &str, + opts: &TestOptions<'_>, + view: &dyn BuildView, +) -> Result<(), Box> { + let mut sources = crate::apt::sources::load(Some(ctx.clone()))?; + let mut modified = false; + let mut added_ppas: Vec<(&str, &str)> = Vec::new(); + + for ppa in &opts.ppa { + let (ppa_user, ppa_name) = crate::package_info::split_ppa(ppa)?; + let base_url = crate::package_info::ppa_to_base_url(ppa_user, ppa_name); + if sources.iter().any(|s| s.uri.contains(&base_url)) { + continue; + } + let mut architectures = vec![crate::get_current_arch()]; + if arch != crate::get_current_arch() { + architectures.push(arch.to_string()); + } + sources.push(crate::apt::sources::SourceEntry { + enabled: true, + kind: crate::apt::sources::SourceKind::Deb, + components: vec!["main".to_string()], + architectures, + signed_by: None, + trusted: None, + suite: vec![series.to_string()], + uri: base_url, + origin: None, + }); + added_ppas.push((ppa_user, ppa_name)); + modified = true; + } + + // Ubuntu: test dependencies live in universe as often as in main. + let ubuntu_components = crate::distro_info::get_dist_components("ubuntu")?; + if ubuntu_components.iter().any(|c| c == "universe") { + for source in &mut sources { + if crate::distro_info::is_official_source("ubuntu", &source.uri) + && !source.components.contains(&"universe".to_string()) + { + source.components.push("universe".to_string()); + modified = true; + } + } + } + + if let Some(pocket) = &opts.pocket { + let pocket_suite = format!("{series}-{pocket}"); + log::info!("Enabling pocket '{}' for test dependencies", pocket_suite); + for source in &mut sources { + if crate::deb::is_archive_source(&source.uri) && !source.suite.contains(&pocket_suite) { + source.suite.push(pocket_suite.clone()); + modified = true; + } + } + // proposed is NotAutomatic: without a pin apt would ignore it. + if pocket.starts_with("proposed") { + crate::deb::local::pin_pocket(&pocket_suite, ctx)?; + } + } + + if modified { + crate::apt::sources::save(Some(ctx.clone()), sources)?; + for (user, ppa_name) in added_ppas { + if let Err(e) = + crate::apt::keyring::download_trust_ppa_key(Some(ctx.clone()), user, ppa_name).await + { + log::warn!("Failed to download PPA key for {user}/{ppa_name}: {e}"); + } + } + } + + view.phase("Configuring testbed", Box::new(GenericClassifier::new())); + let status = ctx + .command("apt-get") + .envs(env.clone()) + .arg("update") + .status() + .map_err(|e| format!("apt-get update failed inside the testbed: {e}"))?; + if !status.success() { + return Err("apt-get update failed inside the testbed".into()); + } + + if opts.apt_upgrade { + for action in ["upgrade", "autopurge"] { + let status = ctx + .command("apt-get") + .envs(env.clone()) + .arg("-y") + .arg(action) + .status() + .map_err(|e| e.to_string())?; + if !status.success() { + return Err(format!("apt-get {action} failed inside the testbed").into()); + } + } + } + + for setup in &opts.setup_commands { + let status = ctx + .command("bash") + .envs(env.clone()) + .arg("-e") + .arg("-c") + .arg(setup) + .status() + .map_err(|e| e.to_string())?; + if !status.success() { + return Err(format!("setup command failed: {setup}").into()); + } + } + + if !opts.inject.is_empty() { + crate::deb::local::install_injected_packages(&opts.inject, env, ctx.clone(), view, &None)?; + } + Ok(()) +} + +/// Drop into an interactive testbed shell (the `--shell`/ +/// `--shell-fail` debugging escape hatch). +fn drop_shell( + bed: &Bed, + staged_tree: &str, + env: &HashMap, + view: &dyn BuildView, +) -> Result<(), Box> { + view.suspend(); + log::info!("Dropping into the testbed shell (exit to continue)"); + let status = bed + .ctx() + .command("bash") + .envs(env.iter()) + .current_dir(staged_tree) + .status()?; + if !status.success() { + log::warn!("Testbed shell exited with {}", status); + } + Ok(()) +} + +/// Create the run's host artifacts directory: +/// `/var/tmp/pkh/tests/---/`. +fn create_run_dir(package: &str, series: &str, arch: &str) -> std::io::Result { + let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%S"); + let dir = + PathBuf::from("/var/tmp/pkh/tests").join(format!("{package}-{series}-{arch}-{timestamp}")); + fs::create_dir_all(&dir)?; + Ok(dir) +} + +/// The binary package name of a staged .deb file name. +fn deb_binary_name(path: &Path) -> Option { + let name = path.file_name()?.to_string_lossy(); + let stem = name.strip_suffix(".deb")?; + Some(stem.split('_').next()?.to_string()) +} + +/// Live terminal passthrough of captured lines, for `--verbose` runs +/// without a live view. +struct TerminalLines; + +impl LineSink for TerminalLines { + fn line(&self, stream: Stream, line: &str) { + match stream { + Stream::Stdout => println!("{line}"), + Stream::Stderr => eprintln!("{line}"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The freshness guard accepts the changes file matching the current + /// changelog entry with its debs, and rejects a stale one. + #[test] + fn fresh_deb_output_freshness() { + let base = tempfile::tempdir().unwrap(); + let tree = base.path().join("hello"); + fs::create_dir_all(tree.join("debian")).unwrap(); + fs::write( + tree.join("debian/changelog"), + "hello (2.10-4) noble; urgency=medium\n\n * bump\n\n -- A Thu, 01 Jan 2026 00:00:00 +0000\n", + ) + .unwrap(); + + let changes_name = "hello_2.10-4_amd64.changes"; + let write_changes = |fresh: bool| { + let content = format!( + "Format: 1.8\nSource: hello\nVersion: 2.10-4\nChecksums-Sha256:\n \ + abc 123 utils {} \n", + changes_name.replace(".changes", ".deb") + ); + let path = base.path().join(changes_name); + fs::write(&path, content).unwrap(); + if !fresh { + // Push the tree strictly newer than the changes file. + let newer = tree.join("debian/control"); + fs::write(&newer, "Source: hello\n").unwrap(); + let time = std::time::SystemTime::now() + std::time::Duration::from_secs(3600); + let file = fs::File::options().append(true).open(&newer).unwrap(); + file.set_times(std::fs::FileTimes::new().set_modified(time)) + .unwrap(); + } + }; + + // No changes file at all: nothing to reuse. + assert!(fresh_deb_output(&tree, "hello", "2.10-4", "amd64").is_none()); + + // The deb the changes file lists must exist for the reuse. + fs::write(base.path().join("hello_2.10-4_amd64.deb"), "deb").unwrap(); + + write_changes(true); + let debs = fresh_deb_output(&tree, "hello", "2.10-4", "amd64").unwrap(); + assert_eq!(debs, vec![base.path().join("hello_2.10-4_amd64.deb")]); + + // A different version or arch tests nothing. + assert!(fresh_deb_output(&tree, "hello", "2.10-3", "amd64").is_none()); + assert!(fresh_deb_output(&tree, "hello", "2.10-4", "arm64").is_none()); + + // Tree content newer than the output makes it stale. + write_changes(false); + assert!(fresh_deb_output(&tree, "hello", "2.10-4", "amd64").is_none()); + } + + /// The run directory is created fresh under the pkh tests root. + #[test] + fn run_dir_layout() { + let dir = create_run_dir("hello", "noble", "amd64").unwrap(); + assert_eq!(dir.parent(), Some(Path::new("/var/tmp/pkh/tests"))); + assert!( + dir.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("hello-noble-amd64-")) + ); + assert!(dir.is_dir()); + fs::remove_dir(&dir).ok(); + } +} diff --git a/src/test/options.rs b/src/test/options.rs new file mode 100644 index 0000000..edea088 --- /dev/null +++ b/src/test/options.rs @@ -0,0 +1,204 @@ +//! Options of a test run (`pkh test`), and their pure resolution rules. + +use crate::report::BuildView; +use std::path::PathBuf; + +/// Testbed mode: where the tests run. +/// +/// The default is [`TestMode::Chroot`]: `pkh deb` defaults to the local +/// context because builds are non-destructive, but autopkgtests install +/// packages and are destructive by design — isolation is the default and +/// running on the host is the explicit choice. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum TestMode { + /// An ephemeral unshare chroot, bootstrapped like a `pkh deb` build + /// chroot (shared tarball cache). + #[default] + Chroot, + /// The current context directly: no isolation, tests can rewrite it. + Local, +} + +impl TestMode { + /// Label used in the report and the view status. + pub fn label(&self) -> &'static str { + match self { + TestMode::Chroot => "chroot", + TestMode::Local => "local", + } + } + + /// Parse a `--mode` value. + pub fn parse(value: &str) -> Option { + match value { + "chroot" => Some(TestMode::Chroot), + "local" => Some(TestMode::Local), + _ => None, + } + } +} + +/// Parameters of one [`crate::test::run`] call. +pub struct TestOptions<'a> { + /// Source tree to test; defaults to the process working directory. + pub tree: Option, + /// Target distribution series; defaults to the changelog series + /// (UNRELEASED resolves to the vendor's development series). + pub series: Option, + /// Target distribution (debian/ubuntu); defaults to the host vendor. + pub dist: Option, + /// Target architecture; defaults to the host architecture. + pub arch: Option, + /// Distribution pocket to resolve test dependencies from. + pub pocket: Option, + /// PPAs to add as dependency sources (`user/ppa_name`). + pub ppa: Vec, + /// Packages to inject into the testbed before the tests (.deb paths + /// or archive names). + pub inject: Vec, + /// Testbed mode; defaults to [`TestMode::Chroot`]. + pub mode: Option, + /// Explicit binary packages to test; overrides the `pkh deb` output + /// reuse. + pub debs: Vec, + /// Never build: error out when no fresh `pkh deb` output exists. + pub no_build: bool, + /// Ignore fresh `pkh deb` output and rebuild through the deb flow. + pub build: bool, + /// Run only these tests (exact names); empty = all. + pub test_names: Vec, + /// Never run these tests (they are reported as skipped). + pub skip_tests: Vec, + /// Extra shell commands run in the testbed after the APT setup. + pub setup_commands: Vec, + /// Upgrade the testbed (update/upgrade/autopurge) before the tests. + pub apt_upgrade: bool, + /// Drop into an interactive testbed shell after the run. + pub shell: bool, + /// Drop into an interactive testbed shell when a test failed. + pub shell_fail: bool, + /// Per-test timeout in seconds; defaults to autopkgtest's 10 000 s, + /// scaled by `timeout_factor`. + pub timeout: Option, + /// Scale the default timeouts (`--timeout-factor`). + pub timeout_factor: Option, + /// Exit 1 when tests were skipped or flaky (`--fail-on skip`). + pub fail_on_skip: bool, + /// Exit 1 when no tests ran (`--fail-on no-tests`). + pub fail_on_no_tests: bool, + /// Render the report as JSON instead of text. + pub json: bool, + /// Raw test output instead of the summarized live view. + pub verbose: bool, + /// Where run events (phases, progress, outcome) are reported. + pub view: &'a dyn BuildView, +} + +impl Default for TestOptions<'_> { + fn default() -> Self { + static QUIET: crate::report::Quiet = crate::report::Quiet; + TestOptions { + tree: None, + series: None, + dist: None, + arch: None, + pocket: None, + ppa: Vec::new(), + inject: Vec::new(), + mode: None, + debs: Vec::new(), + no_build: false, + build: false, + test_names: Vec::new(), + skip_tests: Vec::new(), + setup_commands: Vec::new(), + apt_upgrade: false, + shell: false, + shell_fail: false, + timeout: None, + timeout_factor: None, + fail_on_skip: false, + fail_on_no_tests: false, + json: false, + verbose: false, + view: &QUIET, + } + } +} + +impl TestOptions<'_> { + /// Whether a declared test is selected for the run: an explicit + /// `--test-name` list must contain it, and no `--skip-test` may. + pub fn is_selected(&self, name: &str) -> bool { + (self.test_names.is_empty() || self.test_names.iter().any(|n| n == name)) + && !self.skip_tests.iter().any(|n| n == name) + } + + /// The per-test timeout in seconds: an explicit `--timeout` stands as + /// given; otherwise the autopkgtest default is scaled by + /// `--timeout-factor`. + pub fn timeout_secs(&self) -> u64 { + match (self.timeout, self.timeout_factor) { + (Some(secs), _) => secs, + (None, factor) => (10_000_f64 * factor.unwrap_or(1.0)) as u64, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn options( + test_names: &[&str], + skip_tests: &[&str], + timeout: Option, + factor: Option, + ) -> TestOptions<'static> { + TestOptions { + test_names: test_names.iter().map(|s| s.to_string()).collect(), + skip_tests: skip_tests.iter().map(|s| s.to_string()).collect(), + timeout, + timeout_factor: factor, + ..Default::default() + } + } + + #[test] + fn selection_filters() { + let all = options(&[], &[], None, None); + assert!(all.is_selected("smoke")); + + let only = options(&["smoke", "lint"], &[], None, None); + assert!(only.is_selected("smoke")); + assert!(!only.is_selected("other")); + + let skipped = options(&[], &["heavy"], None, None); + assert!(skipped.is_selected("smoke")); + assert!(!skipped.is_selected("heavy")); + + // An explicit list and a skip compose: the skip wins. + let both = options(&["smoke", "heavy"], &["heavy"], None, None); + assert!(both.is_selected("smoke")); + assert!(!both.is_selected("heavy")); + } + + #[test] + fn timeout_resolution() { + // autopkgtest's default test timeout, unscaled. + assert_eq!(options(&[], &[], None, None).timeout_secs(), 10_000); + // The factor scales only the default. + assert_eq!(options(&[], &[], None, Some(2.0)).timeout_secs(), 20_000); + // An explicit timeout stands as given. + assert_eq!(options(&[], &[], Some(60), Some(2.0)).timeout_secs(), 60); + } + + #[test] + fn mode_parsing() { + assert_eq!(TestMode::parse("chroot"), Some(TestMode::Chroot)); + assert_eq!(TestMode::parse("local"), Some(TestMode::Local)); + assert_eq!(TestMode::parse("vm"), None); + assert_eq!(TestMode::default(), TestMode::Chroot); + assert_eq!(TestMode::Local.label(), "local"); + } +} diff --git a/src/test/outcome.rs b/src/test/outcome.rs new file mode 100644 index 0000000..6b434b6 --- /dev/null +++ b/src/test/outcome.rs @@ -0,0 +1,332 @@ +//! The test run report: per-test outcomes, text/JSON rendering and the +//! pkh exit-code contract. + +use std::path::PathBuf; +use std::time::Duration; + +/// Outcome of one DEP-8 test. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Verdict { + /// The test passed (exit 0, no unexpected stderr). + Pass, + /// The test failed (nonzero exit, unexpected stderr, timeout, or its + /// dependencies could not be installed). + Fail, + /// The test did not run: unmet restriction on this testbed, filtered + /// by `--skip-test`, or self-skipped via exit 77 under `skippable`. + Skip, + /// A `flaky`-restricted test failed; the failure is tolerated (like a + /// skip for the exit code) but reported. + Flaky, +} + +impl Verdict { + /// Lowercase name used in the JSON report and the summary. + pub fn as_str(&self) -> &'static str { + match self { + Verdict::Pass => "pass", + Verdict::Fail => "fail", + Verdict::Skip => "skip", + Verdict::Flaky => "flaky", + } + } +} + +/// One test's recorded outcome. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TestOutcome { + /// Test name (the `Tests:` path or resolved `Test-Command` name). + pub name: String, + /// How the test ended. + pub verdict: Verdict, + /// Skip or failure reason, when there is one. + pub reason: Option, + /// Wall-clock time the test ran for (zero for skipped tests). + pub duration: Duration, +} + +/// The whole run's result, as returned by [`crate::test::run`]. +#[derive(Debug, Clone)] +pub struct TestRunReport { + /// Source package name. + pub package: String, + /// Changelog version the run tested. + pub version: String, + /// Distribution series the testbed ran. + pub series: String, + /// Architecture the testbed ran. + pub arch: String, + /// Testbed mode used (`chroot`, `local`, `vm`). + pub mode: String, + /// The binary packages installed for the tests (host-side paths). + pub binaries: Vec, + /// Where the binaries came from, for the report (freshness reuse, + /// in-process build, explicit request). + pub binaries_note: String, + /// One outcome per selected test, in declaration order. + pub outcomes: Vec, + /// Host directory holding the run's artifacts (log, per-test + /// artifacts). + pub artifacts_dir: PathBuf, + /// Raw autopkgtest exit code, when the run went through the vm + /// escape hatch. + pub raw_exit_code: Option, +} + +impl TestRunReport { + /// Whether at least one test failed. + pub fn has_failures(&self) -> bool { + self.outcomes + .iter() + .any(|outcome| outcome.verdict == Verdict::Fail) + } + + /// The pkh exit code: 0 pass/skip/flaky (unless `--fail-on`), 1 test + /// failure (or `--fail-on` trigger), 2 for runtime errors (those + /// abort the run with an `Err` before a report exists). + pub fn exit_code(&self, fail_on_skip: bool, fail_on_no_tests: bool) -> i32 { + if self.has_failures() { + return 1; + } + if fail_on_skip + && self + .outcomes + .iter() + .any(|outcome| matches!(outcome.verdict, Verdict::Skip | Verdict::Flaky)) + { + return 1; + } + if fail_on_no_tests && self.outcomes.is_empty() { + return 1; + } + 0 + } + + /// Human-readable text report: one line per test, then the summary. + pub fn render_text(&self) -> String { + let mut out = String::new(); + for outcome in &self.outcomes { + let secs = outcome.duration.as_secs_f32(); + match outcome.verdict { + Verdict::Pass => out.push_str(&format!("✓ {} ({secs:.1}s)\n", outcome.name)), + Verdict::Fail => { + out.push_str(&format!("✗ {}", outcome.name)); + if let Some(reason) = &outcome.reason { + out.push_str(&format!(" — {reason}")); + } + out.push_str(&format!(" ({secs:.1}s)\n")); + } + Verdict::Skip => { + out.push_str(&format!("- {} (skipped", outcome.name)); + if let Some(reason) = &outcome.reason { + out.push_str(&format!(": {reason}")); + } + out.push_str(")\n"); + } + Verdict::Flaky => { + out.push_str(&format!("~ {} (flaky", outcome.name)); + if let Some(reason) = &outcome.reason { + out.push_str(&format!(": {reason}")); + } + out.push_str(&format!(") ({secs:.1}s)\n")); + } + } + } + + let pass = self + .outcomes + .iter() + .filter(|outcome| outcome.verdict == Verdict::Pass) + .count(); + let fail = self + .outcomes + .iter() + .filter(|outcome| outcome.verdict == Verdict::Fail) + .count(); + let skipped = self.outcomes.len() - pass - fail; + if self.outcomes.is_empty() { + out.push_str("No tests ran.\n"); + } else if fail > 0 { + out.push_str(&format!( + "{fail} of {} tests failed{}.\n", + self.outcomes.len(), + if skipped > 0 { + format!(" ({skipped} skipped or flaky)") + } else { + String::new() + } + )); + } else { + out.push_str(&format!( + "All {} tests passed{}.\n", + self.outcomes.len(), + if skipped > 0 { + format!(" ({skipped} skipped or flaky)") + } else { + String::new() + } + )); + } + out.push_str(&format!("Artifacts: {}\n", self.artifacts_dir.display())); + out + } + + /// Schema-versioned JSON report, same content as the text rendering. + pub fn render_json(&self) -> String { + #[derive(serde::Serialize)] + #[serde(rename_all = "snake_case")] + struct Outcome<'a> { + name: &'a str, + verdict: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + reason: Option<&'a String>, + duration_secs: f64, + } + #[derive(serde::Serialize)] + #[serde(rename_all = "snake_case")] + struct Report<'a> { + schema: &'static str, + package: &'a str, + version: &'a str, + series: &'a str, + arch: &'a str, + mode: &'a str, + binaries: Vec, + binaries_note: &'a str, + outcomes: Vec>, + artifacts_dir: String, + #[serde(skip_serializing_if = "Option::is_none")] + raw_exit_code: Option, + } + let report = Report { + schema: "pkh-test-report-v1", + package: &self.package, + version: &self.version, + series: &self.series, + arch: &self.arch, + mode: &self.mode, + binaries: self + .binaries + .iter() + .map(|path| path.display().to_string()) + .collect(), + binaries_note: &self.binaries_note, + outcomes: self + .outcomes + .iter() + .map(|outcome| Outcome { + name: &outcome.name, + verdict: outcome.verdict.as_str(), + reason: outcome.reason.as_ref(), + duration_secs: outcome.duration.as_secs_f64(), + }) + .collect(), + artifacts_dir: self.artifacts_dir.display().to_string(), + raw_exit_code: self.raw_exit_code, + }; + serde_json::to_string_pretty(&report).unwrap_or_else(|_| "{}".to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn outcome(name: &str, verdict: Verdict, reason: Option<&str>) -> TestOutcome { + TestOutcome { + name: name.to_string(), + verdict, + reason: reason.map(String::from), + duration: Duration::from_millis(1500), + } + } + + fn report(outcomes: Vec) -> TestRunReport { + TestRunReport { + package: "hello".to_string(), + version: "2.10-4".to_string(), + series: "noble".to_string(), + arch: "amd64".to_string(), + mode: "chroot".to_string(), + binaries: vec![PathBuf::from("../hello_2.10-4_amd64.deb")], + binaries_note: "fresh pkh deb output".to_string(), + outcomes, + artifacts_dir: PathBuf::from("/var/tmp/pkh/tests/hello-noble-amd64-x/"), + raw_exit_code: None, + } + } + + /// The exit-code contract: failures dominate, `--fail-on` promotes + /// skips and empty runs. + #[test] + fn exit_code_matrix() { + let pass = report(vec![outcome("a", Verdict::Pass, None)]); + assert_eq!(pass.exit_code(false, false), 0); + assert_eq!(pass.exit_code(true, true), 0); + + let skip = report(vec![ + outcome("a", Verdict::Pass, None), + outcome("b", Verdict::Skip, Some("needs root")), + ]); + assert_eq!(skip.exit_code(false, false), 0); + assert_eq!(skip.exit_code(true, false), 1); + + let fail = report(vec![ + outcome("a", Verdict::Pass, None), + outcome("b", Verdict::Fail, Some("exit code 1")), + ]); + assert_eq!(fail.exit_code(false, false), 1); + // A failure dominates every other trigger. + assert_eq!(fail.exit_code(true, true), 1); + + let empty = report(vec![]); + assert_eq!(empty.exit_code(false, false), 0); + assert_eq!(empty.exit_code(false, true), 1); + + let flaky = report(vec![outcome("a", Verdict::Flaky, Some("passed on retry"))]); + assert_eq!(flaky.exit_code(false, false), 0); + assert_eq!(flaky.exit_code(true, false), 1); + } + + /// The text rendering shows every verdict with reasons and a summary. + #[test] + fn text_rendering() { + let run = report(vec![ + outcome("smoke", Verdict::Pass, None), + outcome("l10n", Verdict::Fail, Some("exit code 1")), + outcome("heavy", Verdict::Skip, Some("requires a VM")), + ]); + let text = run.render_text(); + assert!(text.contains("✓ smoke"), "{text}"); + assert!(text.contains("✗ l10n — exit code 1"), "{text}"); + assert!(text.contains("- heavy (skipped: requires a VM)"), "{text}"); + assert!( + text.contains("1 of 3 tests failed (1 skipped or flaky)"), + "{text}" + ); + assert!( + text.contains("Artifacts: /var/tmp/pkh/tests/hello-noble-amd64-x/"), + "{text}" + ); + + let empty = report(vec![]).render_text(); + assert!(empty.contains("No tests ran."), "{empty}"); + + let clean = report(vec![outcome("smoke", Verdict::Pass, None)]).render_text(); + assert!(clean.contains("All 1 tests passed."), "{clean}"); + } + + /// The JSON report round-trips the report shape with a schema marker. + #[test] + fn json_rendering() { + let run = report(vec![outcome("smoke", Verdict::Pass, None)]); + let json: serde_json::Value = serde_json::from_str(&run.render_json()).unwrap(); + assert_eq!(json["schema"], "pkh-test-report-v1"); + assert_eq!(json["package"], "hello"); + assert_eq!(json["mode"], "chroot"); + assert_eq!(json["outcomes"][0]["verdict"], "pass"); + assert_eq!(json["outcomes"][0]["duration_secs"], 1.5); + assert!(json.get("reason").is_none() && json["outcomes"][0].get("reason").is_none()); + assert!(json.get("raw_exit_code").is_none()); + } +} diff --git a/src/test/runner.rs b/src/test/runner.rs new file mode 100644 index 0000000..524fa8b --- /dev/null +++ b/src/test/runner.rs @@ -0,0 +1,544 @@ +//! The DEP-8 per-test contract: restriction gates, dependency +//! installation, the run environment, verdicts and artifact retrieval. +//! +//! The contract (autopkgtest 6.x `README.package-tests`): tests run in +//! the unpacked source tree with a fresh `$AUTOPKGTEST_TMP`; exit 0 with +//! empty stderr passes, any stderr output fails unless the +//! `allow-stderr` restriction is declared, exit 77 skips under +//! `skippable`, and a per-test timeout kills the test. + +use std::collections::HashMap; +use std::fs; +use std::io::Write; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use crate::context::{Context, LineSink, Stream}; +use crate::debian::control::ControlInfo; +use crate::debian::tests_control::Test; +use crate::logfmt::GenericClassifier; +use crate::report::BuildView; +use crate::test::expand::expand_depends; +use crate::test::outcome::{TestOutcome, Verdict}; + +/// Grace period `timeout --kill-after` gives a test after SIGTERM. +const KILL_AFTER_SECS: u64 = 15; + +/// Everything one test run needs from the orchestration. +pub(crate) struct RunContext<'a> { + /// The testbed context. + pub ctx: Arc, + /// Context-side path of the staged source tree. + pub tree: String, + /// Context-side scratch root (temporary directories, artifacts). + pub staging_root: String, + /// The testbed architecture. + pub arch: String, + /// The host architecture (for `skip-foreign-architecture`). + pub host_arch: String, + /// Whether root-requiring tests can run. + pub supports_root: bool, + /// Per-test timeout in seconds. + pub timeout_secs: u64, + /// The view's tee sink, when the view consumes subprocess lines. + pub sink: Option>, + /// Where run events are reported. + pub view: &'a dyn BuildView, + /// The source control file, for `Depends:` expansion. + pub control: ControlInfo, + /// The binary package names built for the tests. + pub binaries: Vec, + /// Base environment for testbed commands. + pub env: HashMap, + /// Host directory holding the run's artifacts. + pub artifacts_dir: PathBuf, +} + +/// Run the tests in declaration order, filtered by the caller's +/// selection, and return one outcome per selected test. +pub(crate) fn run_tests( + run: &RunContext<'_>, + tests: &[Test], + only: &[String], + skipped: &[String], +) -> Vec { + let selected: Vec<&Test> = tests + .iter() + .filter(|test| only.is_empty() || only.iter().any(|name| name == &test.name)) + .collect(); + let total = selected.len(); + + run.view + .phase("Running tests", Box::new(GenericClassifier::new())); + let mut outcomes = Vec::new(); + for (index, test) in selected.into_iter().enumerate() { + let outcome = if skipped.iter().any(|name| name == &test.name) { + TestOutcome { + name: test.name.clone(), + verdict: Verdict::Skip, + reason: Some("excluded by --skip-test".to_string()), + duration: Duration::ZERO, + } + } else { + run_one(run, test) + }; + run.view.progress("Tests", index + 1, total); + outcomes.push(outcome); + } + outcomes +} + +/// The result of one raw test execution, as seen by the verdict logic. +pub(crate) struct RawRun { + /// Process exit code (`None` when killed by a signal). + pub code: Option, + /// Whether the test produced any stderr output. + pub stderr_nonempty: bool, + /// Whether the `timeout` wrapper killed the test. + pub timed_out: bool, +} + +/// The verdict for one raw run, per the DEP-8 contract. +pub(crate) fn decide(test: &Test, raw: &RawRun, timeout_secs: u64) -> (Verdict, Option) { + if raw.timed_out { + return ( + Verdict::Fail, + Some(format!("timed out after {timeout_secs}s")), + ); + } + match raw.code { + // `timeout` reports its kill with 124; a test exiting 124 by + // itself is indistinguishable and counted as a timeout. + Some(124) => ( + Verdict::Fail, + Some(format!("timed out after {timeout_secs}s")), + ), + None => (Verdict::Fail, Some("killed by a signal".to_string())), + // A self-skipping test may be chatty: the skip verdict comes + // before the stderr rule. + Some(77) if test.has_restriction("skippable") => ( + Verdict::Skip, + Some("the test skipped itself (exit 77)".to_string()), + ), + Some(0) if !raw.stderr_nonempty || test.has_restriction("allow-stderr") => { + (Verdict::Pass, None) + } + Some(0) => (Verdict::Fail, Some("the test wrote to stderr".to_string())), + Some(code) => (Verdict::Fail, Some(format!("exit code {code}"))), + } +} + +/// The static reason a test cannot run on this testbed, if any. +pub(crate) fn restriction_skip( + test: &Test, + supports_root: bool, + foreign_arch: bool, +) -> Option { + if test.has_restriction("isolation-machine") { + return Some("requires a virtual machine (isolation-machine); use --mode vm".to_string()); + } + if test.has_restriction("needs-reboot") { + return Some("requires reboot support (needs-reboot); use --mode vm".to_string()); + } + if test.has_restriction("isolation-container") { + return Some("requires container isolation (isolation-container)".to_string()); + } + if test.has_restriction("skip-foreign-architecture") && foreign_arch { + return Some("skipped on a foreign architecture (skip-foreign-architecture)".to_string()); + } + if (test.has_restriction("needs-root") || test.has_restriction("needs-sudo")) && !supports_root + { + return Some("requires root on this testbed".to_string()); + } + None +} + +/// Run one test through the DEP-8 contract. +fn run_one(run: &RunContext<'_>, test: &Test) -> TestOutcome { + let started = Instant::now(); + if let Some(reason) = restriction_skip(test, run.supports_root, run.arch != run.host_arch) { + return TestOutcome { + name: test.name.clone(), + verdict: Verdict::Skip, + reason: Some(reason), + duration: Duration::ZERO, + }; + } + + // Test dependencies: expanded per stanza, installed before the test. + if let Err(reason) = install_test_deps(run, test) { + return outcome_fail(test, started, reason); + } + + // build-needed: the tests need the tree in its built state. + if test.has_restriction("build-needed") + && let Err(reason) = build_tree(run) + { + return outcome_fail(test, started, reason); + } + + let (raw, duration) = execute(run, test); + let (mut verdict, mut reason) = decide(test, &raw, run.timeout_secs); + + // flaky: one tolerated failure is retried once and reported as FLAKY + // rather than FAIL (autopkgtest's semantics). + if verdict == Verdict::Fail && test.has_restriction("flaky") { + let (retry_raw, _) = execute(run, test); + let (retry_verdict, retry_reason) = decide(test, &retry_raw, run.timeout_secs); + reason = if retry_verdict == Verdict::Pass { + Some("passed on retry".to_string()) + } else { + reason.or(retry_reason) + }; + verdict = Verdict::Flaky; + } + + TestOutcome { + name: test.name.clone(), + verdict, + reason, + duration, + } +} + +fn outcome_fail(test: &Test, started: Instant, reason: String) -> TestOutcome { + TestOutcome { + name: test.name.clone(), + verdict: Verdict::Fail, + reason: Some(reason), + duration: started.elapsed(), + } +} + +/// Execute a test once, returning the raw result and its duration. +fn execute(run: &RunContext<'_>, test: &Test) -> (RawRun, Duration) { + let started = Instant::now(); + + // Fresh per-test scratch and artifact directories. + let tmp_dir = run + .ctx + .create_temp_dir() + .unwrap_or_else(|_| format!("{}/tmp-{}", run.staging_root, safe_name(&test.name))); + let artifacts_dir = format!("{}/artifacts/{}", run.staging_root, safe_name(&test.name)); + let _ = run + .ctx + .command("mkdir") + .arg("-p") + .arg(&artifacts_dir) + .status(); + + // The test's argv, wrapped in the per-test timeout. + let mut argv: Vec = vec![ + "timeout".to_string(), + "--foreground".to_string(), + format!("--kill-after={KILL_AFTER_SECS}"), + run.timeout_secs.to_string(), + ]; + match (&test.script, &test.command) { + (Some(script), _) => { + let path = test_script_path(run, test, script); + // Scripts may be shipped without the executable bit. + let _ = run.ctx.command("chmod").arg("+x").arg(&path).status(); + argv.push(path); + } + (None, Some(command)) => { + argv.push("bash".to_string()); + argv.push("-e".to_string()); + argv.push("-c".to_string()); + argv.push(command.clone()); + } + (None, None) => unreachable!("parser rejects stanzas with neither Tests nor Test-Command"), + } + + // The DEP-8 environment. AUTOPKGTEST_* are the current names; the + // ADT_* aliases are still read by older tests. + let mut env = run.env.clone(); + for (key, value) in [ + ("AUTOPKGTEST_TMP", tmp_dir.as_str()), + ("ADTTMP", tmp_dir.as_str()), + ("AUTOPKGTEST_ARTIFACTS", artifacts_dir.as_str()), + ("ADT_ARTIFACTS", artifacts_dir.as_str()), + ("AUTOPKGTEST_TESTBED_ARCH", run.arch.as_str()), + ] { + env.insert(key.to_string(), value.to_string()); + } + + let capture = Arc::new(TestSink::new( + host_artifacts_dir(run, &test.name), + run.sink.clone(), + )); + let mut cmd = run.ctx.command(&argv[0]); + cmd.args(&argv[1..]) + .envs(env.iter()) + .current_dir(&run.tree) + .capture(capture.clone()); + let status = cmd.status().ok(); + + // The scratch directory is disposable; the artifacts directory was + // already retrieved by the sink's host-side files plus the copy + // below (tests may have written into $AUTOPKGTEST_ARTIFACTS). + let _ = run.ctx.command("rm").arg("-rf").arg(&tmp_dir).status(); + retrieve_artifacts(run, &test.name); + + let raw = RawRun { + code: status.as_ref().and_then(|status| status.code()), + stderr_nonempty: capture.stderr_lines() > 0, + timed_out: status.as_ref().and_then(|status| status.code()) == Some(124), + }; + (raw, started.elapsed()) +} + +/// Install a test stanza's expanded dependencies. +fn install_test_deps(run: &RunContext<'_>, test: &Test) -> Result<(), String> { + let specs = expand_depends( + test.depends.as_deref(), + &run.control, + &run.binaries, + &run.arch, + ) + .map_err(|e| format!("cannot expand the test dependencies: {e}"))?; + if specs.is_empty() { + return Ok(()); + } + run.view + .message(&format!("Installing dependencies of {}", test.name)); + let mut cmd = run.ctx.command("apt-get"); + cmd.envs(run.env.clone()) + .arg("-y") + .arg("--no-install-recommends") + .arg("install"); + for spec in &specs { + cmd.arg(spec); + } + let status = cmd.status().map_err(|e| e.to_string())?; + if status.success() { + Ok(()) + } else { + Err(format!( + "dependency installation failed ({})", + specs.join(", ") + )) + } +} + +/// Bring the staged tree to its built state (`build-needed` tests): the +/// autopkgtest build, with the nocheck profile — the tests need the +/// build products, not another set of binaries. +fn build_tree(run: &RunContext<'_>) -> Result<(), String> { + let mut env = run.env.clone(); + env.insert("DEB_BUILD_OPTIONS".to_string(), "nocheck".to_string()); + let mut cmd = run.ctx.command("dpkg-buildpackage"); + cmd.envs(env.iter()) + .arg("-us") + .arg("-uc") + .arg("-b") + .current_dir(&run.tree); + let status = cmd.status().map_err(|e| e.to_string())?; + if status.success() { + Ok(()) + } else { + Err("building the source tree failed (build-needed)".to_string()) + } +} + +/// The context-side path of a test script: `Tests-Directory:` replaces +/// the `debian/tests` path segment. +fn test_script_path(run: &RunContext<'_>, test: &Test, script: &str) -> String { + let base = test.tests_directory.as_deref().unwrap_or("debian/tests"); + format!("{}/{base}/{script}", run.tree) +} + +/// Retrieve a test's artifacts directory from the testbed into the run's +/// host directory (best-effort: tests may not have written anything). +fn retrieve_artifacts(run: &RunContext<'_>, test_name: &str) { + let remote = PathBuf::from(format!( + "{}/artifacts/{}", + run.staging_root, + safe_name(test_name) + )); + if !run.ctx.exists(&remote).unwrap_or(false) { + return; + } + let host = host_artifacts_dir(run, test_name); + let _ = fs::create_dir_all(&host); + let _ = run.ctx.retrieve_path(&remote, &host); +} + +/// The host-side artifacts directory of one test. +fn host_artifacts_dir(run: &RunContext<'_>, test_name: &str) -> PathBuf { + run.artifacts_dir + .join("artifacts") + .join(safe_name(test_name)) +} + +/// Filesystem-safe directory name for a test name (script paths may +/// contain slashes). +fn safe_name(name: &str) -> String { + name.replace('/', "_") +} + +/// Per-test capture: appends stdout/stderr lines to files in the run's +/// artifacts directory, counts stderr output (the DEP-8 failure rule) +/// and forwards every line to the view's tee sink. +struct TestSink { + stdout: Mutex, + stderr: Mutex, + stderr_lines: AtomicUsize, + forward: Option>, +} + +impl TestSink { + /// Build the sink; the log files live under the test's artifacts + /// directory on the host. + fn new(host_dir: PathBuf, forward: Option>) -> TestSink { + let _ = fs::create_dir_all(&host_dir); + let open = |name: &str| { + fs::OpenOptions::new() + .create(true) + .append(true) + .open(host_dir.join(name)) + .unwrap_or_else(|_| { + // No artifacts directory: capture into a null file. + fs::File::create("/dev/null").expect("/dev/null is always openable") + }) + }; + TestSink { + stdout: Mutex::new(open("stdout.log")), + stderr: Mutex::new(open("stderr.log")), + stderr_lines: AtomicUsize::new(0), + forward, + } + } + + /// Number of stderr lines seen. + fn stderr_lines(&self) -> usize { + self.stderr_lines.load(Ordering::Relaxed) + } +} + +impl LineSink for TestSink { + fn line(&self, stream: Stream, line: &str) { + let mut file = match stream { + Stream::Stdout => self + .stdout + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + Stream::Stderr => { + self.stderr_lines.fetch_add(1, Ordering::Relaxed); + self.stderr + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + }; + let _ = writeln!(file, "{line}"); + if let Some(forward) = &self.forward { + forward.line(stream, line); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test(restrictions: &[&str]) -> Test { + Test { + name: "smoke".to_string(), + script: Some("smoke".to_string()), + command: None, + depends: None, + restrictions: restrictions.iter().map(|s| s.to_string()).collect(), + tests_directory: None, + } + } + + fn raw(code: Option, stderr: bool) -> RawRun { + RawRun { + code, + stderr_nonempty: stderr, + timed_out: false, + } + } + + /// The verdict matrix: exit/stderr/skip/timeout rules. + #[test] + fn verdict_matrix() { + let plain = test(&[]); + assert_eq!( + decide(&plain, &raw(Some(0), false), 10), + (Verdict::Pass, None) + ); + assert_eq!( + decide(&plain, &raw(Some(0), true), 10), + (Verdict::Fail, Some("the test wrote to stderr".to_string())) + ); + assert_eq!( + decide(&plain, &raw(Some(1), false), 10), + (Verdict::Fail, Some("exit code 1".to_string())) + ); + assert_eq!( + decide(&plain, &raw(None, false), 10), + (Verdict::Fail, Some("killed by a signal".to_string())) + ); + assert_eq!( + decide(&plain, &raw(Some(124), false), 42), + (Verdict::Fail, Some("timed out after 42s".to_string())) + ); + + // exit 77 skips only under `skippable` — and a chatty + // self-skipping test still skips (the verdict comes first). + assert_eq!(decide(&plain, &raw(Some(77), false), 10).0, Verdict::Fail); + let skippable = test(&["skippable"]); + assert_eq!( + decide(&skippable, &raw(Some(77), true), 10).0, + Verdict::Skip + ); + + // allow-stderr lifts the stderr rule. + let chatty = test(&["allow-stderr"]); + assert_eq!(decide(&chatty, &raw(Some(0), true), 10).0, Verdict::Pass); + } + + /// Restriction gates: machine isolation and root requirements skip + /// with an actionable reason. + #[test] + fn restriction_gates() { + let plain = test(&[]); + assert_eq!(restriction_skip(&plain, true, false), None); + assert_eq!(restriction_skip(&plain, false, true), None); + + let vm_only = test(&["isolation-machine"]); + assert!( + restriction_skip(&vm_only, true, false) + .unwrap() + .contains("--mode vm") + ); + + let reboot = test(&["needs-reboot"]); + assert!(restriction_skip(&reboot, true, false).is_some()); + + let container = test(&["isolation-container"]); + assert!(restriction_skip(&container, true, false).is_some()); + + let root = test(&["needs-root"]); + assert!(restriction_skip(&root, false, false).is_some()); + assert_eq!(restriction_skip(&root, true, false), None); + + let sudo = test(&["needs-sudo"]); + assert!(restriction_skip(&sudo, false, false).is_some()); + + let native_only = test(&["skip-foreign-architecture"]); + assert!(restriction_skip(&native_only, true, true).is_some()); + assert_eq!(restriction_skip(&native_only, true, false), None); + } + + /// Test names map to filesystem-safe artifact directory names. + #[test] + fn names_are_sanitized() { + assert_eq!(safe_name("checks/cli"), "checks_cli"); + assert_eq!(safe_name("smoke"), "smoke"); + } +}