test: add pkh test, the native DEP-8 runner for local and chroot
The DEP-8 run-time contract is one page, so pkh implements it natively instead of wrapping autopkgtest (see plans/pkh-test.md): the testbed is a pkh context — chroot mode reuses the deb ephemeral chroots and their cached mmdebstrap tarballs, local mode runs on the current context — and the binaries under test are the freshness-guarded pkh deb output next to the tree, rebuilt through the deb flow when stale or missing. Per test: restrictions gate on the testbed (isolation-machine, needs-reboot and container isolation skip with a pointer at --mode vm), the stanza Depends are expanded (@, @builddeps@, @recommends@) and installed, then the script or Test-Command runs in the staged tree with the AUTOPKGTEST_* (and legacy ADT_*) environment, a per-test timeout and the stderr/exit-77 verdict rules; flaky tests retry once. stdout, stderr and $AUTOPKGTEST_ARTIFACTS are collected per test under /var/tmp/pkh/tests/ (pruned separately). --shell/--shell-fail drop into the testbed for debugging. pin_pocket and install_injected_packages become crate-internal helpers shared with the deb flow, which they mirror.
This commit is contained in:
+2
-2
@@ -1075,7 +1075,7 @@ fn options_declare_single_debian_patch(local_options: Option<&str>, options: Opt
|
||||
/// suite on every repository carrying it (archive, security and ports),
|
||||
/// for all architectures, so this also covers cross-builds pulling
|
||||
/// dependencies from 'ports.ubuntu.com'.
|
||||
fn pin_pocket(pocket_suite: &str, ctx: &Arc<Context>) -> Result<(), Box<dyn Error>> {
|
||||
pub(crate) fn pin_pocket(pocket_suite: &str, ctx: &Arc<Context>) -> Result<(), Box<dyn Error>> {
|
||||
let pin_path = format!("/etc/apt/preferences.d/pkh-{}", pocket_suite);
|
||||
let pin_content = format!(
|
||||
"Package: *\nPin: release a={}\nPin-Priority: 600\n",
|
||||
@@ -1086,7 +1086,7 @@ fn pin_pocket(pocket_suite: &str, ctx: &Arc<Context>) -> Result<(), Box<dyn Erro
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn install_injected_packages(
|
||||
pub(crate) fn install_injected_packages(
|
||||
packages: &[String],
|
||||
env: &HashMap<String, String>,
|
||||
ctx: Arc<Context>,
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ mod cross;
|
||||
/// Ephemeral (per-build) unshare contexts, including the process-global
|
||||
/// cleanup-hook registry drained by the SIGINT handler
|
||||
pub(crate) mod ephemeral;
|
||||
mod local;
|
||||
pub(crate) mod local;
|
||||
/// Resumable build sessions: manifest, discovery, `pkh deb list` and the
|
||||
/// host-tree snapshot/sync that keeps build artifacts alive across attempts
|
||||
pub mod session;
|
||||
|
||||
@@ -38,6 +38,8 @@ pub mod pull;
|
||||
pub mod put;
|
||||
/// Handle package-specific quirks and workarounds
|
||||
pub mod quirks;
|
||||
/// Run a source package's DEP-8 as-installed tests (`pkh test`)
|
||||
pub mod test;
|
||||
|
||||
/// Line classifiers rewriting raw subprocess output into display actions
|
||||
/// and countable progress (pure logic, shared by build views)
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
//! Testbed lifecycle: where and how the tests run.
|
||||
//!
|
||||
//! A testbed is a [`Context`] plus its teardown. chroot mode reuses the
|
||||
//! `pkh deb` machinery wholesale (an ephemeral unshare context guarded by
|
||||
//! [`crate::deb::ephemeral::EphemeralContextGuard`], bootstrapped from the
|
||||
//! same cached mmdebstrap tarballs); local mode is the current context
|
||||
//! used directly. The testbed itself is disposable per run: autopkgtests
|
||||
//! may break it, and the next run starts from a clean bootstrap.
|
||||
|
||||
use std::error::Error;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::context::Context;
|
||||
use crate::deb::ephemeral::EphemeralContextGuard;
|
||||
use crate::report::BuildView;
|
||||
use crate::test::options::TestMode;
|
||||
|
||||
/// A prepared testbed, torn down on drop.
|
||||
pub(crate) enum Bed {
|
||||
/// An ephemeral unshare chroot.
|
||||
Chroot(EphemeralContextGuard),
|
||||
/// The current context, used directly.
|
||||
Local(Arc<Context>),
|
||||
}
|
||||
|
||||
impl Bed {
|
||||
/// Prepare a testbed of `mode` for `series`/`arch` on `base_ctx`.
|
||||
pub(crate) async fn prepare(
|
||||
mode: TestMode,
|
||||
series: &str,
|
||||
arch: &str,
|
||||
base_ctx: Arc<Context>,
|
||||
view: &dyn BuildView,
|
||||
) -> Result<Bed, Box<dyn Error>> {
|
||||
match mode {
|
||||
TestMode::Chroot => {
|
||||
// Mirror the deb flow: only a foreign target requests a
|
||||
// foreign-arch chroot, so the native case shares deb's
|
||||
// unsuffixed `<series>-buildd.tar.xz` cache entry.
|
||||
let chroot_arch = (arch != crate::get_current_arch()).then(|| arch.to_string());
|
||||
let guard = EphemeralContextGuard::new_with_context(
|
||||
series,
|
||||
chroot_arch.as_deref(),
|
||||
base_ctx,
|
||||
view,
|
||||
)
|
||||
.await?;
|
||||
Ok(Bed::Chroot(guard))
|
||||
}
|
||||
TestMode::Local => Ok(Bed::Local(base_ctx)),
|
||||
}
|
||||
}
|
||||
|
||||
/// The context to run testbed commands in.
|
||||
pub(crate) fn ctx(&self) -> Arc<Context> {
|
||||
match self {
|
||||
Bed::Chroot(guard) => guard.context(),
|
||||
Bed::Local(ctx) => Arc::clone(ctx),
|
||||
}
|
||||
}
|
||||
|
||||
/// A scratch root inside the testbed for the staged tree, test
|
||||
/// temporary directories and artifacts.
|
||||
pub(crate) fn scratch_root(&self) -> Result<String, Box<dyn Error>> {
|
||||
match self {
|
||||
// A stable root inside the chroot; the guard creates a fresh
|
||||
// temp directory for anonymous chroots.
|
||||
Bed::Chroot(guard) => guard.build_root(),
|
||||
Bed::Local(ctx) => Ok(ctx.create_temp_dir()?),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether root-requiring tests can run: always inside a chroot
|
||||
/// (commands run as mapped root), on the local context only when the
|
||||
/// context itself runs as root.
|
||||
pub(crate) fn supports_root(&self) -> bool {
|
||||
match self {
|
||||
Bed::Chroot(_) => true,
|
||||
Bed::Local(ctx) => matches!(
|
||||
ctx.command("id")
|
||||
.arg("-u")
|
||||
.output()
|
||||
.map(|output| String::from_utf8_lossy(&output.stdout).trim().to_string()),
|
||||
Ok(id) if id == "0"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
//! DEP-8 `Depends:` substitution: `@`, `@builddeps@` and `@recommends@`
|
||||
//! expanded into concrete package specs for `apt-get install`.
|
||||
//!
|
||||
//! The built binaries themselves are installed separately (from the
|
||||
//! staged `.deb` files, so tests exercise exactly what pkh built);
|
||||
//! expansion substitutes bare binary *names*, which apt then resolves
|
||||
//! against the installed binaries first and the archive second.
|
||||
|
||||
use crate::debian::control::ControlInfo;
|
||||
use crate::debian::deps::{Deps, ParseOpts};
|
||||
|
||||
/// Expand a test stanza's `Depends:` value into apt package specs.
|
||||
///
|
||||
/// `raw` is the value as written (`None` = absent, the DEP-8 default
|
||||
/// `@`). The substitutions, per DEP-8:
|
||||
///
|
||||
/// - `@` in an or-clause replicates the clause once per binary package
|
||||
/// built from the source, with the name substituted (other
|
||||
/// alternatives and version constraints are kept);
|
||||
/// - `@builddeps@` becomes the source's Build-Depends,
|
||||
/// Build-Depends-Indep and Build-Depends-Arch plus `build-essential`;
|
||||
/// - `@recommends@` becomes the Recommends of every binary package,
|
||||
/// with `${...}` substvar references dropped (autopkgtest's
|
||||
/// "variables stripped") but versions kept.
|
||||
///
|
||||
/// Returns one apt spec per comma-separated clause, in order; an empty
|
||||
/// result means nothing to install.
|
||||
pub fn expand_depends(
|
||||
raw: Option<&str>,
|
||||
control: &ControlInfo,
|
||||
binaries: &[String],
|
||||
host_arch: &str,
|
||||
) -> Result<Vec<String>, String> {
|
||||
let raw = raw.unwrap_or("@");
|
||||
let mut specs = Vec::new();
|
||||
for token in split_top_level_commas(raw) {
|
||||
match token.as_str() {
|
||||
"@builddeps@" => {
|
||||
for field in ["Build-Depends", "Build-Depends-Indep", "Build-Depends-Arch"] {
|
||||
if let Some(value) = non_empty(control.source.get(field)) {
|
||||
for clause in split_top_level_commas(value) {
|
||||
if !clause.is_empty() {
|
||||
specs.push(clause);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
specs.push("build-essential".to_string());
|
||||
}
|
||||
"@recommends@" => {
|
||||
for paragraph in &control.binaries {
|
||||
if let Some(value) = non_empty(paragraph.get("Recommends")) {
|
||||
specs.extend(name_only_clauses(value, host_arch)?);
|
||||
}
|
||||
}
|
||||
}
|
||||
token if token.split_whitespace().any(|word| word == "@") => {
|
||||
// Textual substitution: DEP-8 @ is a word-level
|
||||
// placeholder the deps grammar does not know. The clause
|
||||
// replicates once per binary with the name substituted,
|
||||
// constraints and alternatives kept verbatim.
|
||||
for binary in binaries {
|
||||
specs.push(
|
||||
token
|
||||
.split_whitespace()
|
||||
.map(|word| {
|
||||
if word == "@" {
|
||||
binary.clone()
|
||||
} else {
|
||||
word.to_string()
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(" "),
|
||||
);
|
||||
}
|
||||
}
|
||||
token => specs.push(token.to_string()),
|
||||
}
|
||||
}
|
||||
Ok(specs)
|
||||
}
|
||||
|
||||
/// Split on commas that sit outside any parentheses: or-clauses carry
|
||||
/// their own commas inside `(...)` constraints, top-level commas separate
|
||||
/// dependencies.
|
||||
fn split_top_level_commas(input: &str) -> Vec<String> {
|
||||
let mut parts = Vec::new();
|
||||
let mut depth = 0usize;
|
||||
let mut current = String::new();
|
||||
for character in input.chars() {
|
||||
match character {
|
||||
'(' => {
|
||||
depth += 1;
|
||||
current.push(character);
|
||||
}
|
||||
')' => {
|
||||
depth = depth.saturating_sub(1);
|
||||
current.push(character);
|
||||
}
|
||||
',' if depth == 0 => {
|
||||
parts.push(current.trim().to_string());
|
||||
current.clear();
|
||||
}
|
||||
_ => current.push(character),
|
||||
}
|
||||
}
|
||||
parts.push(current.trim().to_string());
|
||||
parts
|
||||
}
|
||||
|
||||
/// Render each or-clause of a dependency list, dropping alternatives
|
||||
/// that are `${...}` substvar references (autopkgtest's "variables
|
||||
/// stripped") and keeping everything else verbatim.
|
||||
fn name_only_clauses(input: &str, host_arch: &str) -> Result<Vec<String>, String> {
|
||||
let opts = ParseOpts::build_deps(host_arch.to_string(), Vec::new());
|
||||
let deps = Deps::parse(input, &opts)?;
|
||||
let mut clauses = Vec::new();
|
||||
for clause in deps.clauses() {
|
||||
let kept: Vec<String> = clause
|
||||
.iter()
|
||||
.filter(|relation| !relation.package.contains('$'))
|
||||
.map(|relation| relation.output())
|
||||
.collect();
|
||||
if !kept.is_empty() {
|
||||
clauses.push(kept.join(" | "));
|
||||
}
|
||||
}
|
||||
Ok(clauses)
|
||||
}
|
||||
|
||||
/// A field value with its surrounding whitespace trimmed; `None` when the
|
||||
/// field is absent or whitespace-only.
|
||||
fn non_empty(value: Option<&str>) -> Option<&str> {
|
||||
value.map(str::trim).filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
/// The binary package names declared by a control file, in stanza order.
|
||||
pub fn binary_names(control: &ControlInfo) -> Vec<String> {
|
||||
control
|
||||
.binaries
|
||||
.iter()
|
||||
.filter_map(|paragraph| paragraph.get("Package"))
|
||||
.map(|name| name.trim().to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::str::FromStr;
|
||||
|
||||
fn control() -> ControlInfo {
|
||||
ControlInfo::from_str(
|
||||
"Source: hello\n\
|
||||
Build-Depends: debhelper-compat (= 13),\n meson\n\
|
||||
Build-Depends-Indep: dh-python\n\
|
||||
\n\
|
||||
Package: hello\n\
|
||||
Architecture: any\n\
|
||||
Recommends: hello-doc, pulseaudio (>= 15)\n\
|
||||
\n\
|
||||
Package: hello-doc\n\
|
||||
Architecture: all\n\
|
||||
Recommends: hello\n",
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn binaries() -> Vec<String> {
|
||||
vec!["hello".to_string(), "hello-doc".to_string()]
|
||||
}
|
||||
|
||||
/// The default (absent) Depends is `@`: one spec per binary.
|
||||
#[test]
|
||||
fn default_is_at() {
|
||||
let specs = expand_depends(None, &control(), &binaries(), "amd64").unwrap();
|
||||
assert_eq!(specs, ["hello", "hello-doc"]);
|
||||
}
|
||||
|
||||
/// An or-clause containing @ replicates once per binary, keeping the
|
||||
/// other alternatives and version constraints.
|
||||
#[test]
|
||||
fn or_clauses_replicate_per_binary() {
|
||||
let specs = expand_depends(
|
||||
Some("@ (>= 1.0) | fallback, jq"),
|
||||
&control(),
|
||||
&binaries(),
|
||||
"amd64",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
specs,
|
||||
[
|
||||
"hello (>= 1.0) | fallback",
|
||||
"hello-doc (>= 1.0) | fallback",
|
||||
"jq"
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// @builddeps@ pulls the three Build-* fields plus build-essential.
|
||||
#[test]
|
||||
fn builddeps_expansion() {
|
||||
let specs =
|
||||
expand_depends(Some("@builddeps@, @"), &control(), &binaries(), "amd64").unwrap();
|
||||
assert_eq!(
|
||||
specs,
|
||||
[
|
||||
"debhelper-compat (= 13)",
|
||||
"meson",
|
||||
"dh-python",
|
||||
"build-essential",
|
||||
"hello",
|
||||
"hello-doc"
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// @recommends@ collects every binary's Recommends, stripped to names.
|
||||
#[test]
|
||||
fn recommends_expansion() {
|
||||
let specs = expand_depends(Some("@recommends@"), &control(), &binaries(), "amd64").unwrap();
|
||||
assert_eq!(specs, ["hello-doc", "pulseaudio (>= 15)", "hello"]);
|
||||
}
|
||||
|
||||
/// Plain dependencies pass through untouched; a token mentioning @ in
|
||||
/// a version constraint is still a substitution.
|
||||
#[test]
|
||||
fn plain_tokens_pass_through() {
|
||||
let specs = expand_depends(
|
||||
Some("ca-certificates, python3:any"),
|
||||
&control(),
|
||||
&binaries(),
|
||||
"amd64",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(specs, ["ca-certificates", "python3:any"]);
|
||||
}
|
||||
|
||||
/// Commas inside version constraints do not split clauses.
|
||||
#[test]
|
||||
fn parenthesised_commas_stay_together() {
|
||||
let specs =
|
||||
expand_depends(Some("foo (>= 1) , bar"), &control(), &binaries(), "amd64").unwrap();
|
||||
assert_eq!(specs, ["foo (>= 1)", "bar"]);
|
||||
}
|
||||
}
|
||||
+628
@@ -0,0 +1,628 @@
|
||||
//! Run a source package's DEP-8 as-installed tests (`pkh test`).
|
||||
//!
|
||||
//! The runner is native for the chroot and local testbeds: the testbed
|
||||
//! is a pkh context (an ephemeral unshare chroot bootstrapped exactly
|
||||
//! like a `pkh deb` build chroot, or the current context directly), and
|
||||
//! the DEP-8 per-test contract lives in [`runner`]. `--mode vm` execs
|
||||
//! the installed autopkgtest with its qemu runner instead — the one
|
||||
//! backend pkh does not reimplement (serial-console protocol, image
|
||||
//! management, reboot support).
|
||||
//!
|
||||
//! Binaries under test are the fresh `pkh deb` output next to the tree
|
||||
//! (freshness-guarded like the lint wrapper); when they are missing or
|
||||
//! stale, the deb flow builds them in-process first.
|
||||
|
||||
pub(crate) mod bed;
|
||||
pub mod expand;
|
||||
pub mod options;
|
||||
pub mod outcome;
|
||||
pub(crate) mod runner;
|
||||
|
||||
pub use options::{TestMode, TestOptions};
|
||||
pub use outcome::TestRunReport;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::context::{LineSink, Stream};
|
||||
use crate::debian::control::{ControlInfo, parse_paragraphs};
|
||||
use crate::debian::tests_control::TestsControl;
|
||||
use crate::logfmt::GenericClassifier;
|
||||
use crate::report::{BuildTarget, BuildView, TargetKind};
|
||||
use crate::test::bed::Bed;
|
||||
use crate::test::outcome::Verdict;
|
||||
|
||||
/// Where the built binaries under test came from.
|
||||
struct Binaries {
|
||||
/// Host-side paths of the .deb files.
|
||||
paths: Vec<PathBuf>,
|
||||
/// Provenance note for the report.
|
||||
note: String,
|
||||
}
|
||||
|
||||
/// Run the DEP-8 tests of a source package tree.
|
||||
///
|
||||
/// Errors are runtime errors (no declared tests, unusable testbed,
|
||||
/// failed in-process build); test failures are a report, not an error.
|
||||
pub async fn run(opts: TestOptions<'_>) -> Result<TestRunReport, Box<dyn std::error::Error>> {
|
||||
let view = opts.view;
|
||||
let tree = opts
|
||||
.tree
|
||||
.clone()
|
||||
.unwrap_or_else(|| std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")));
|
||||
let tree = tree
|
||||
.canonicalize()
|
||||
.map_err(|e| format!("cannot access the source tree '{}': {e}", tree.display()))?;
|
||||
|
||||
// Package identity and target, resolved like a deb build.
|
||||
let entry = crate::debian::parse_changelog_entry(&tree.join("debian/changelog"))
|
||||
.map_err(|e| format!("cannot read the package identity: {e}"))?;
|
||||
let package = entry.source.clone();
|
||||
let version = entry.version.full();
|
||||
let dist = opts
|
||||
.dist
|
||||
.clone()
|
||||
.unwrap_or_else(crate::build::env::current_vendor);
|
||||
let series = match opts.series.clone() {
|
||||
Some(series) => series,
|
||||
None if crate::distro_info::is_unreleased(&entry.distribution) => {
|
||||
let resolved = crate::distro_info::effective_series(&entry.distribution, &dist).await?;
|
||||
log::info!(
|
||||
"Changelog is UNRELEASED, testing against series {}",
|
||||
resolved
|
||||
);
|
||||
resolved
|
||||
}
|
||||
None => entry.distribution.clone(),
|
||||
};
|
||||
let arch = opts.arch.clone().unwrap_or_else(crate::get_current_arch);
|
||||
|
||||
// Pre-check: the package must declare tests, before any testbed is
|
||||
// booted.
|
||||
let control_path = tree.join("debian/tests/control");
|
||||
if !control_path.exists() {
|
||||
return Err(
|
||||
"the package declares no autopkgtests (no debian/tests/control). \
|
||||
Add debian/tests/control and 'Testsuite: autopkgtest' to debian/control \
|
||||
(pkh lint flags this too), or point pkh test at another tree."
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
let tests_control = TestsControl::parse(&control_path)
|
||||
.map_err(|e| format!("cannot parse debian/tests/control: {e}"))?;
|
||||
if tests_control.tests.is_empty() {
|
||||
return Err("debian/tests/control declares no tests".into());
|
||||
}
|
||||
|
||||
// Binaries under test: fresh pkh deb output, an in-process build, or
|
||||
// an explicit --debs list.
|
||||
let binaries = resolve_binaries(&opts, &tree, &package, &version, &arch, view).await?;
|
||||
|
||||
view.target(BuildTarget {
|
||||
package: &package,
|
||||
version: &version,
|
||||
target: &format!("{series}/{arch}"),
|
||||
display: format!(
|
||||
"Testing {} ({}) on {}/{} ({})",
|
||||
package,
|
||||
version,
|
||||
series,
|
||||
arch,
|
||||
opts.mode.unwrap_or_default().label()
|
||||
),
|
||||
kind: TargetKind::Test,
|
||||
tee_log: true,
|
||||
});
|
||||
|
||||
let mode = opts.mode.unwrap_or_default();
|
||||
if mode == TestMode::Local {
|
||||
log::warn!(
|
||||
"Tests run directly on the current context: they install packages \
|
||||
and can rewrite it. Use the default chroot mode for isolation."
|
||||
);
|
||||
}
|
||||
let run_dir = create_run_dir(&package, &series, &arch)?;
|
||||
let sink = view.sink().or_else(|| {
|
||||
// Verbose runs without a live view still stream test output live.
|
||||
opts.verbose
|
||||
.then(|| Arc::new(TerminalLines) as Arc<dyn LineSink>)
|
||||
});
|
||||
|
||||
let base_ctx = crate::context::current();
|
||||
let bed = Bed::prepare(mode, &series, &arch, base_ctx, view).await?;
|
||||
let ctx = bed.ctx();
|
||||
|
||||
let mut env = HashMap::new();
|
||||
env.insert("LANG".to_string(), "C".to_string());
|
||||
env.insert("DEBIAN_FRONTEND".to_string(), "noninteractive".to_string());
|
||||
if matches!(bed, Bed::Chroot(_)) {
|
||||
env.insert("HOME".to_string(), "/root".to_string());
|
||||
env.insert("USER".to_string(), "root".to_string());
|
||||
env.insert("LOGNAME".to_string(), "root".to_string());
|
||||
}
|
||||
|
||||
configure_testbed(&ctx, &env, &series, &arch, &opts, view).await?;
|
||||
|
||||
// Stage the tree and the binaries into the testbed. The copy also
|
||||
// protects the host tree: tests may write into it (rw-build-tree).
|
||||
view.message("Staging the tree and binaries");
|
||||
let staging_root = bed.scratch_root()?;
|
||||
let staged_tree = ctx.ensure_available(&tree, &staging_root)?;
|
||||
let staged_tree = staged_tree.to_string_lossy().into_owned();
|
||||
let mut staged_debs = Vec::new();
|
||||
for deb in &binaries.paths {
|
||||
staged_debs.push(ctx.ensure_available(deb, &staging_root)?);
|
||||
}
|
||||
|
||||
// Install the exact binaries under test before the tests run.
|
||||
if !staged_debs.is_empty() {
|
||||
view.phase(
|
||||
"Installing built binaries",
|
||||
Box::new(GenericClassifier::new()),
|
||||
);
|
||||
let mut cmd = ctx.command("apt-get");
|
||||
cmd.envs(env.clone())
|
||||
.arg("-y")
|
||||
.arg("--allow-downgrades")
|
||||
.arg("install");
|
||||
for deb in &staged_debs {
|
||||
cmd.arg(format!(
|
||||
"./{}",
|
||||
deb.to_string_lossy().trim_start_matches('/')
|
||||
));
|
||||
}
|
||||
let status = cmd.status().map_err(|e| e.to_string())?;
|
||||
if !status.success() {
|
||||
return Err("installing the built binaries into the testbed failed".into());
|
||||
}
|
||||
}
|
||||
|
||||
// What @ expands to: the names of the binaries actually under test,
|
||||
// from their .deb file names (<binary>_<version>_<arch>.deb).
|
||||
let control_info = ControlInfo::parse(&tree.join("debian/control"))
|
||||
.map_err(|e| format!("cannot parse debian/control: {e}"))?;
|
||||
let binary_names: Vec<String> = staged_debs
|
||||
.iter()
|
||||
.filter_map(|deb| deb_binary_name(deb))
|
||||
.collect();
|
||||
let run_ctx = runner::RunContext {
|
||||
ctx,
|
||||
tree: staged_tree.clone(),
|
||||
staging_root,
|
||||
arch: arch.clone(),
|
||||
host_arch: crate::get_current_arch(),
|
||||
supports_root: bed.supports_root(),
|
||||
timeout_secs: opts.timeout_secs(),
|
||||
sink,
|
||||
view,
|
||||
control: control_info,
|
||||
binaries: binary_names,
|
||||
env: env.clone(),
|
||||
artifacts_dir: run_dir.clone(),
|
||||
};
|
||||
let outcomes = runner::run_tests(
|
||||
&run_ctx,
|
||||
&tests_control.tests,
|
||||
&opts.test_names,
|
||||
&opts.skip_tests,
|
||||
);
|
||||
|
||||
// Debugging escape hatches.
|
||||
let has_failures = outcomes
|
||||
.iter()
|
||||
.any(|outcome| outcome.verdict == Verdict::Fail);
|
||||
if opts.shell || (opts.shell_fail && has_failures) {
|
||||
drop_shell(&bed, &staged_tree, &env, view)?;
|
||||
}
|
||||
|
||||
let report = TestRunReport {
|
||||
package,
|
||||
version,
|
||||
series,
|
||||
arch,
|
||||
mode: mode.label().to_string(),
|
||||
binaries: binaries.paths,
|
||||
binaries_note: binaries.note,
|
||||
outcomes,
|
||||
artifacts_dir: run_dir.clone(),
|
||||
raw_exit_code: None,
|
||||
};
|
||||
if report.has_failures() {
|
||||
view.finish_failure();
|
||||
} else {
|
||||
view.finish_success(&[run_dir]);
|
||||
}
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
/// Resolve the binaries under test: `--debs` overrides everything;
|
||||
/// otherwise the fresh `pkh deb` output next to the tree (freshness-
|
||||
/// guarded, the lint wrapper's artifact strategy); otherwise the deb
|
||||
/// flow builds them in-process. `--no-build` refuses instead.
|
||||
async fn resolve_binaries(
|
||||
opts: &TestOptions<'_>,
|
||||
tree: &Path,
|
||||
package: &str,
|
||||
version: &str,
|
||||
arch: &str,
|
||||
view: &dyn BuildView,
|
||||
) -> Result<Binaries, Box<dyn std::error::Error>> {
|
||||
if !opts.debs.is_empty() {
|
||||
for deb in &opts.debs {
|
||||
if !deb.exists() {
|
||||
return Err(format!("--deb file '{}' does not exist", deb.display()).into());
|
||||
}
|
||||
}
|
||||
return Ok(Binaries {
|
||||
paths: opts.debs.clone(),
|
||||
note: "explicitly passed with --debs".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
if !opts.build
|
||||
&& let Some(paths) = fresh_deb_output(tree, package, version, arch)
|
||||
{
|
||||
log::info!("Testing the fresh pkh deb output next to the tree");
|
||||
return Ok(Binaries {
|
||||
paths,
|
||||
note: "fresh pkh deb output".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
if opts.no_build {
|
||||
return Err(
|
||||
"no fresh pkh deb output next to the tree and --no-build given: \
|
||||
run 'pkh deb' first, drop --no-build, or pass --debs"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
|
||||
view.message("No fresh pkh deb output: building the binaries first");
|
||||
let artifacts = crate::deb::build_binary_package(crate::deb::DebBuildOptions {
|
||||
series: opts.series.clone(),
|
||||
arch: opts.arch.clone(),
|
||||
pocket: opts.pocket.clone(),
|
||||
ppa: opts.ppa.clone(),
|
||||
cwd: Some(tree.to_path_buf()),
|
||||
view,
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let debs: Vec<PathBuf> = artifacts
|
||||
.into_iter()
|
||||
.filter(|path| path.extension().is_some_and(|ext| ext == "deb"))
|
||||
.collect();
|
||||
if debs.is_empty() {
|
||||
return Err("the build produced no binary packages to test".into());
|
||||
}
|
||||
Ok(Binaries {
|
||||
paths: debs,
|
||||
note: "built by pkh deb just now".to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The fresh `pkh deb` output next to the tree, when it matches the
|
||||
/// current changelog entry and no tree content is newer than it: the
|
||||
/// `<source>_<version>_<arch>.changes` file gates, its checksum lists
|
||||
/// name the debs. Stale or missing output tests nothing.
|
||||
fn fresh_deb_output(tree: &Path, package: &str, version: &str, arch: &str) -> Option<Vec<PathBuf>> {
|
||||
let entry = crate::debian::parse_changelog_entry(&tree.join("debian/changelog")).ok()?;
|
||||
let parent = tree.parent()?;
|
||||
let changes_path = parent.join(format!(
|
||||
"{}_{}_{}.changes",
|
||||
package,
|
||||
entry.version.no_epoch(),
|
||||
arch
|
||||
));
|
||||
let changes = fs::read_to_string(&changes_path).ok()?;
|
||||
|
||||
// Only the current entry's binaries are the ones under test.
|
||||
let paragraph = parse_paragraphs(&changes).into_iter().next()?;
|
||||
let matches = paragraph.get("Source").map(str::trim) == Some(package)
|
||||
&& paragraph.get("Version").map(str::trim) == Some(version);
|
||||
if !matches {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut debs = Vec::new();
|
||||
for (field, value) in paragraph.iter() {
|
||||
if !field.starts_with("Checksums-") && field != "Files" {
|
||||
continue;
|
||||
}
|
||||
for line in value.lines() {
|
||||
// `<checksum> <size> <section> [<priority>] <filename>`
|
||||
if let Some(name) = line.split_whitespace().last()
|
||||
&& name.ends_with(".deb")
|
||||
{
|
||||
debs.push(parent.join(name));
|
||||
}
|
||||
}
|
||||
}
|
||||
if debs.is_empty() || debs.iter().any(|deb| !deb.exists()) {
|
||||
return None;
|
||||
}
|
||||
// A tree file written after the changes file makes the output stale.
|
||||
let built = fs::metadata(&changes_path).ok()?.modified().ok()?;
|
||||
if tree_newer_than(tree, built) {
|
||||
return None;
|
||||
}
|
||||
Some(debs)
|
||||
}
|
||||
|
||||
/// Whether any tree content is newer than `built`. Skips `.git` and
|
||||
/// `.pc`: commits and quilt bookkeeping churn their mtimes without
|
||||
/// touching what the package contains.
|
||||
fn tree_newer_than(root: &Path, built: std::time::SystemTime) -> bool {
|
||||
let mut stack = vec![root.to_path_buf()];
|
||||
while let Some(dir) = stack.pop() {
|
||||
let Ok(entries) = fs::read_dir(&dir) else {
|
||||
continue;
|
||||
};
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
let name = entry.file_name();
|
||||
if name == ".git" || name == ".pc" {
|
||||
continue;
|
||||
}
|
||||
let Ok(meta) = fs::symlink_metadata(&path) else {
|
||||
continue;
|
||||
};
|
||||
if meta.is_dir() {
|
||||
stack.push(path);
|
||||
} else if meta.modified().is_ok_and(|mtime| mtime > built) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Configure the testbed's APT setup and update it: pockets and PPAs on
|
||||
/// the archive sources (the same rules as the deb build), an optional
|
||||
/// upgrade, the caller's setup commands and injected packages.
|
||||
async fn configure_testbed(
|
||||
ctx: &Arc<crate::context::Context>,
|
||||
env: &HashMap<String, String>,
|
||||
series: &str,
|
||||
arch: &str,
|
||||
opts: &TestOptions<'_>,
|
||||
view: &dyn BuildView,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut sources = crate::apt::sources::load(Some(ctx.clone()))?;
|
||||
let mut modified = false;
|
||||
let mut added_ppas: Vec<(&str, &str)> = Vec::new();
|
||||
|
||||
for ppa in &opts.ppa {
|
||||
let (ppa_user, ppa_name) = crate::package_info::split_ppa(ppa)?;
|
||||
let base_url = crate::package_info::ppa_to_base_url(ppa_user, ppa_name);
|
||||
if sources.iter().any(|s| s.uri.contains(&base_url)) {
|
||||
continue;
|
||||
}
|
||||
let mut architectures = vec![crate::get_current_arch()];
|
||||
if arch != crate::get_current_arch() {
|
||||
architectures.push(arch.to_string());
|
||||
}
|
||||
sources.push(crate::apt::sources::SourceEntry {
|
||||
enabled: true,
|
||||
kind: crate::apt::sources::SourceKind::Deb,
|
||||
components: vec!["main".to_string()],
|
||||
architectures,
|
||||
signed_by: None,
|
||||
trusted: None,
|
||||
suite: vec![series.to_string()],
|
||||
uri: base_url,
|
||||
origin: None,
|
||||
});
|
||||
added_ppas.push((ppa_user, ppa_name));
|
||||
modified = true;
|
||||
}
|
||||
|
||||
// Ubuntu: test dependencies live in universe as often as in main.
|
||||
let ubuntu_components = crate::distro_info::get_dist_components("ubuntu")?;
|
||||
if ubuntu_components.iter().any(|c| c == "universe") {
|
||||
for source in &mut sources {
|
||||
if crate::distro_info::is_official_source("ubuntu", &source.uri)
|
||||
&& !source.components.contains(&"universe".to_string())
|
||||
{
|
||||
source.components.push("universe".to_string());
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(pocket) = &opts.pocket {
|
||||
let pocket_suite = format!("{series}-{pocket}");
|
||||
log::info!("Enabling pocket '{}' for test dependencies", pocket_suite);
|
||||
for source in &mut sources {
|
||||
if crate::deb::is_archive_source(&source.uri) && !source.suite.contains(&pocket_suite) {
|
||||
source.suite.push(pocket_suite.clone());
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
// proposed is NotAutomatic: without a pin apt would ignore it.
|
||||
if pocket.starts_with("proposed") {
|
||||
crate::deb::local::pin_pocket(&pocket_suite, ctx)?;
|
||||
}
|
||||
}
|
||||
|
||||
if modified {
|
||||
crate::apt::sources::save(Some(ctx.clone()), sources)?;
|
||||
for (user, ppa_name) in added_ppas {
|
||||
if let Err(e) =
|
||||
crate::apt::keyring::download_trust_ppa_key(Some(ctx.clone()), user, ppa_name).await
|
||||
{
|
||||
log::warn!("Failed to download PPA key for {user}/{ppa_name}: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
view.phase("Configuring testbed", Box::new(GenericClassifier::new()));
|
||||
let status = ctx
|
||||
.command("apt-get")
|
||||
.envs(env.clone())
|
||||
.arg("update")
|
||||
.status()
|
||||
.map_err(|e| format!("apt-get update failed inside the testbed: {e}"))?;
|
||||
if !status.success() {
|
||||
return Err("apt-get update failed inside the testbed".into());
|
||||
}
|
||||
|
||||
if opts.apt_upgrade {
|
||||
for action in ["upgrade", "autopurge"] {
|
||||
let status = ctx
|
||||
.command("apt-get")
|
||||
.envs(env.clone())
|
||||
.arg("-y")
|
||||
.arg(action)
|
||||
.status()
|
||||
.map_err(|e| e.to_string())?;
|
||||
if !status.success() {
|
||||
return Err(format!("apt-get {action} failed inside the testbed").into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for setup in &opts.setup_commands {
|
||||
let status = ctx
|
||||
.command("bash")
|
||||
.envs(env.clone())
|
||||
.arg("-e")
|
||||
.arg("-c")
|
||||
.arg(setup)
|
||||
.status()
|
||||
.map_err(|e| e.to_string())?;
|
||||
if !status.success() {
|
||||
return Err(format!("setup command failed: {setup}").into());
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.inject.is_empty() {
|
||||
crate::deb::local::install_injected_packages(&opts.inject, env, ctx.clone(), view, &None)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drop into an interactive testbed shell (the `--shell`/
|
||||
/// `--shell-fail` debugging escape hatch).
|
||||
fn drop_shell(
|
||||
bed: &Bed,
|
||||
staged_tree: &str,
|
||||
env: &HashMap<String, String>,
|
||||
view: &dyn BuildView,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
view.suspend();
|
||||
log::info!("Dropping into the testbed shell (exit to continue)");
|
||||
let status = bed
|
||||
.ctx()
|
||||
.command("bash")
|
||||
.envs(env.iter())
|
||||
.current_dir(staged_tree)
|
||||
.status()?;
|
||||
if !status.success() {
|
||||
log::warn!("Testbed shell exited with {}", status);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create the run's host artifacts directory:
|
||||
/// `/var/tmp/pkh/tests/<source>-<series>-<arch>-<timestamp>/`.
|
||||
fn create_run_dir(package: &str, series: &str, arch: &str) -> std::io::Result<PathBuf> {
|
||||
let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%S");
|
||||
let dir =
|
||||
PathBuf::from("/var/tmp/pkh/tests").join(format!("{package}-{series}-{arch}-{timestamp}"));
|
||||
fs::create_dir_all(&dir)?;
|
||||
Ok(dir)
|
||||
}
|
||||
|
||||
/// The binary package name of a staged .deb file name.
|
||||
fn deb_binary_name(path: &Path) -> Option<String> {
|
||||
let name = path.file_name()?.to_string_lossy();
|
||||
let stem = name.strip_suffix(".deb")?;
|
||||
Some(stem.split('_').next()?.to_string())
|
||||
}
|
||||
|
||||
/// Live terminal passthrough of captured lines, for `--verbose` runs
|
||||
/// without a live view.
|
||||
struct TerminalLines;
|
||||
|
||||
impl LineSink for TerminalLines {
|
||||
fn line(&self, stream: Stream, line: &str) {
|
||||
match stream {
|
||||
Stream::Stdout => println!("{line}"),
|
||||
Stream::Stderr => eprintln!("{line}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The freshness guard accepts the changes file matching the current
|
||||
/// changelog entry with its debs, and rejects a stale one.
|
||||
#[test]
|
||||
fn fresh_deb_output_freshness() {
|
||||
let base = tempfile::tempdir().unwrap();
|
||||
let tree = base.path().join("hello");
|
||||
fs::create_dir_all(tree.join("debian")).unwrap();
|
||||
fs::write(
|
||||
tree.join("debian/changelog"),
|
||||
"hello (2.10-4) noble; urgency=medium\n\n * bump\n\n -- A <a@b.c> Thu, 01 Jan 2026 00:00:00 +0000\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let changes_name = "hello_2.10-4_amd64.changes";
|
||||
let write_changes = |fresh: bool| {
|
||||
let content = format!(
|
||||
"Format: 1.8\nSource: hello\nVersion: 2.10-4\nChecksums-Sha256:\n \
|
||||
abc 123 utils {} \n",
|
||||
changes_name.replace(".changes", ".deb")
|
||||
);
|
||||
let path = base.path().join(changes_name);
|
||||
fs::write(&path, content).unwrap();
|
||||
if !fresh {
|
||||
// Push the tree strictly newer than the changes file.
|
||||
let newer = tree.join("debian/control");
|
||||
fs::write(&newer, "Source: hello\n").unwrap();
|
||||
let time = std::time::SystemTime::now() + std::time::Duration::from_secs(3600);
|
||||
let file = fs::File::options().append(true).open(&newer).unwrap();
|
||||
file.set_times(std::fs::FileTimes::new().set_modified(time))
|
||||
.unwrap();
|
||||
}
|
||||
};
|
||||
|
||||
// No changes file at all: nothing to reuse.
|
||||
assert!(fresh_deb_output(&tree, "hello", "2.10-4", "amd64").is_none());
|
||||
|
||||
// The deb the changes file lists must exist for the reuse.
|
||||
fs::write(base.path().join("hello_2.10-4_amd64.deb"), "deb").unwrap();
|
||||
|
||||
write_changes(true);
|
||||
let debs = fresh_deb_output(&tree, "hello", "2.10-4", "amd64").unwrap();
|
||||
assert_eq!(debs, vec![base.path().join("hello_2.10-4_amd64.deb")]);
|
||||
|
||||
// A different version or arch tests nothing.
|
||||
assert!(fresh_deb_output(&tree, "hello", "2.10-3", "amd64").is_none());
|
||||
assert!(fresh_deb_output(&tree, "hello", "2.10-4", "arm64").is_none());
|
||||
|
||||
// Tree content newer than the output makes it stale.
|
||||
write_changes(false);
|
||||
assert!(fresh_deb_output(&tree, "hello", "2.10-4", "amd64").is_none());
|
||||
}
|
||||
|
||||
/// The run directory is created fresh under the pkh tests root.
|
||||
#[test]
|
||||
fn run_dir_layout() {
|
||||
let dir = create_run_dir("hello", "noble", "amd64").unwrap();
|
||||
assert_eq!(dir.parent(), Some(Path::new("/var/tmp/pkh/tests")));
|
||||
assert!(
|
||||
dir.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.is_some_and(|name| name.starts_with("hello-noble-amd64-"))
|
||||
);
|
||||
assert!(dir.is_dir());
|
||||
fs::remove_dir(&dir).ok();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
//! Options of a test run (`pkh test`), and their pure resolution rules.
|
||||
|
||||
use crate::report::BuildView;
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// Testbed mode: where the tests run.
|
||||
///
|
||||
/// The default is [`TestMode::Chroot`]: `pkh deb` defaults to the local
|
||||
/// context because builds are non-destructive, but autopkgtests install
|
||||
/// packages and are destructive by design — isolation is the default and
|
||||
/// running on the host is the explicit choice.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum TestMode {
|
||||
/// An ephemeral unshare chroot, bootstrapped like a `pkh deb` build
|
||||
/// chroot (shared tarball cache).
|
||||
#[default]
|
||||
Chroot,
|
||||
/// The current context directly: no isolation, tests can rewrite it.
|
||||
Local,
|
||||
}
|
||||
|
||||
impl TestMode {
|
||||
/// Label used in the report and the view status.
|
||||
pub fn label(&self) -> &'static str {
|
||||
match self {
|
||||
TestMode::Chroot => "chroot",
|
||||
TestMode::Local => "local",
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a `--mode` value.
|
||||
pub fn parse(value: &str) -> Option<TestMode> {
|
||||
match value {
|
||||
"chroot" => Some(TestMode::Chroot),
|
||||
"local" => Some(TestMode::Local),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parameters of one [`crate::test::run`] call.
|
||||
pub struct TestOptions<'a> {
|
||||
/// Source tree to test; defaults to the process working directory.
|
||||
pub tree: Option<PathBuf>,
|
||||
/// Target distribution series; defaults to the changelog series
|
||||
/// (UNRELEASED resolves to the vendor's development series).
|
||||
pub series: Option<String>,
|
||||
/// Target distribution (debian/ubuntu); defaults to the host vendor.
|
||||
pub dist: Option<String>,
|
||||
/// Target architecture; defaults to the host architecture.
|
||||
pub arch: Option<String>,
|
||||
/// Distribution pocket to resolve test dependencies from.
|
||||
pub pocket: Option<String>,
|
||||
/// PPAs to add as dependency sources (`user/ppa_name`).
|
||||
pub ppa: Vec<String>,
|
||||
/// Packages to inject into the testbed before the tests (.deb paths
|
||||
/// or archive names).
|
||||
pub inject: Vec<String>,
|
||||
/// Testbed mode; defaults to [`TestMode::Chroot`].
|
||||
pub mode: Option<TestMode>,
|
||||
/// Explicit binary packages to test; overrides the `pkh deb` output
|
||||
/// reuse.
|
||||
pub debs: Vec<PathBuf>,
|
||||
/// Never build: error out when no fresh `pkh deb` output exists.
|
||||
pub no_build: bool,
|
||||
/// Ignore fresh `pkh deb` output and rebuild through the deb flow.
|
||||
pub build: bool,
|
||||
/// Run only these tests (exact names); empty = all.
|
||||
pub test_names: Vec<String>,
|
||||
/// Never run these tests (they are reported as skipped).
|
||||
pub skip_tests: Vec<String>,
|
||||
/// Extra shell commands run in the testbed after the APT setup.
|
||||
pub setup_commands: Vec<String>,
|
||||
/// Upgrade the testbed (update/upgrade/autopurge) before the tests.
|
||||
pub apt_upgrade: bool,
|
||||
/// Drop into an interactive testbed shell after the run.
|
||||
pub shell: bool,
|
||||
/// Drop into an interactive testbed shell when a test failed.
|
||||
pub shell_fail: bool,
|
||||
/// Per-test timeout in seconds; defaults to autopkgtest's 10 000 s,
|
||||
/// scaled by `timeout_factor`.
|
||||
pub timeout: Option<u64>,
|
||||
/// Scale the default timeouts (`--timeout-factor`).
|
||||
pub timeout_factor: Option<f64>,
|
||||
/// Exit 1 when tests were skipped or flaky (`--fail-on skip`).
|
||||
pub fail_on_skip: bool,
|
||||
/// Exit 1 when no tests ran (`--fail-on no-tests`).
|
||||
pub fail_on_no_tests: bool,
|
||||
/// Render the report as JSON instead of text.
|
||||
pub json: bool,
|
||||
/// Raw test output instead of the summarized live view.
|
||||
pub verbose: bool,
|
||||
/// Where run events (phases, progress, outcome) are reported.
|
||||
pub view: &'a dyn BuildView,
|
||||
}
|
||||
|
||||
impl Default for TestOptions<'_> {
|
||||
fn default() -> Self {
|
||||
static QUIET: crate::report::Quiet = crate::report::Quiet;
|
||||
TestOptions {
|
||||
tree: None,
|
||||
series: None,
|
||||
dist: None,
|
||||
arch: None,
|
||||
pocket: None,
|
||||
ppa: Vec::new(),
|
||||
inject: Vec::new(),
|
||||
mode: None,
|
||||
debs: Vec::new(),
|
||||
no_build: false,
|
||||
build: false,
|
||||
test_names: Vec::new(),
|
||||
skip_tests: Vec::new(),
|
||||
setup_commands: Vec::new(),
|
||||
apt_upgrade: false,
|
||||
shell: false,
|
||||
shell_fail: false,
|
||||
timeout: None,
|
||||
timeout_factor: None,
|
||||
fail_on_skip: false,
|
||||
fail_on_no_tests: false,
|
||||
json: false,
|
||||
verbose: false,
|
||||
view: &QUIET,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TestOptions<'_> {
|
||||
/// Whether a declared test is selected for the run: an explicit
|
||||
/// `--test-name` list must contain it, and no `--skip-test` may.
|
||||
pub fn is_selected(&self, name: &str) -> bool {
|
||||
(self.test_names.is_empty() || self.test_names.iter().any(|n| n == name))
|
||||
&& !self.skip_tests.iter().any(|n| n == name)
|
||||
}
|
||||
|
||||
/// The per-test timeout in seconds: an explicit `--timeout` stands as
|
||||
/// given; otherwise the autopkgtest default is scaled by
|
||||
/// `--timeout-factor`.
|
||||
pub fn timeout_secs(&self) -> u64 {
|
||||
match (self.timeout, self.timeout_factor) {
|
||||
(Some(secs), _) => secs,
|
||||
(None, factor) => (10_000_f64 * factor.unwrap_or(1.0)) as u64,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn options(
|
||||
test_names: &[&str],
|
||||
skip_tests: &[&str],
|
||||
timeout: Option<u64>,
|
||||
factor: Option<f64>,
|
||||
) -> TestOptions<'static> {
|
||||
TestOptions {
|
||||
test_names: test_names.iter().map(|s| s.to_string()).collect(),
|
||||
skip_tests: skip_tests.iter().map(|s| s.to_string()).collect(),
|
||||
timeout,
|
||||
timeout_factor: factor,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selection_filters() {
|
||||
let all = options(&[], &[], None, None);
|
||||
assert!(all.is_selected("smoke"));
|
||||
|
||||
let only = options(&["smoke", "lint"], &[], None, None);
|
||||
assert!(only.is_selected("smoke"));
|
||||
assert!(!only.is_selected("other"));
|
||||
|
||||
let skipped = options(&[], &["heavy"], None, None);
|
||||
assert!(skipped.is_selected("smoke"));
|
||||
assert!(!skipped.is_selected("heavy"));
|
||||
|
||||
// An explicit list and a skip compose: the skip wins.
|
||||
let both = options(&["smoke", "heavy"], &["heavy"], None, None);
|
||||
assert!(both.is_selected("smoke"));
|
||||
assert!(!both.is_selected("heavy"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timeout_resolution() {
|
||||
// autopkgtest's default test timeout, unscaled.
|
||||
assert_eq!(options(&[], &[], None, None).timeout_secs(), 10_000);
|
||||
// The factor scales only the default.
|
||||
assert_eq!(options(&[], &[], None, Some(2.0)).timeout_secs(), 20_000);
|
||||
// An explicit timeout stands as given.
|
||||
assert_eq!(options(&[], &[], Some(60), Some(2.0)).timeout_secs(), 60);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mode_parsing() {
|
||||
assert_eq!(TestMode::parse("chroot"), Some(TestMode::Chroot));
|
||||
assert_eq!(TestMode::parse("local"), Some(TestMode::Local));
|
||||
assert_eq!(TestMode::parse("vm"), None);
|
||||
assert_eq!(TestMode::default(), TestMode::Chroot);
|
||||
assert_eq!(TestMode::Local.label(), "local");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
//! The test run report: per-test outcomes, text/JSON rendering and the
|
||||
//! pkh exit-code contract.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Outcome of one DEP-8 test.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Verdict {
|
||||
/// The test passed (exit 0, no unexpected stderr).
|
||||
Pass,
|
||||
/// The test failed (nonzero exit, unexpected stderr, timeout, or its
|
||||
/// dependencies could not be installed).
|
||||
Fail,
|
||||
/// The test did not run: unmet restriction on this testbed, filtered
|
||||
/// by `--skip-test`, or self-skipped via exit 77 under `skippable`.
|
||||
Skip,
|
||||
/// A `flaky`-restricted test failed; the failure is tolerated (like a
|
||||
/// skip for the exit code) but reported.
|
||||
Flaky,
|
||||
}
|
||||
|
||||
impl Verdict {
|
||||
/// Lowercase name used in the JSON report and the summary.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Verdict::Pass => "pass",
|
||||
Verdict::Fail => "fail",
|
||||
Verdict::Skip => "skip",
|
||||
Verdict::Flaky => "flaky",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One test's recorded outcome.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TestOutcome {
|
||||
/// Test name (the `Tests:` path or resolved `Test-Command` name).
|
||||
pub name: String,
|
||||
/// How the test ended.
|
||||
pub verdict: Verdict,
|
||||
/// Skip or failure reason, when there is one.
|
||||
pub reason: Option<String>,
|
||||
/// Wall-clock time the test ran for (zero for skipped tests).
|
||||
pub duration: Duration,
|
||||
}
|
||||
|
||||
/// The whole run's result, as returned by [`crate::test::run`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TestRunReport {
|
||||
/// Source package name.
|
||||
pub package: String,
|
||||
/// Changelog version the run tested.
|
||||
pub version: String,
|
||||
/// Distribution series the testbed ran.
|
||||
pub series: String,
|
||||
/// Architecture the testbed ran.
|
||||
pub arch: String,
|
||||
/// Testbed mode used (`chroot`, `local`, `vm`).
|
||||
pub mode: String,
|
||||
/// The binary packages installed for the tests (host-side paths).
|
||||
pub binaries: Vec<PathBuf>,
|
||||
/// Where the binaries came from, for the report (freshness reuse,
|
||||
/// in-process build, explicit request).
|
||||
pub binaries_note: String,
|
||||
/// One outcome per selected test, in declaration order.
|
||||
pub outcomes: Vec<TestOutcome>,
|
||||
/// Host directory holding the run's artifacts (log, per-test
|
||||
/// artifacts).
|
||||
pub artifacts_dir: PathBuf,
|
||||
/// Raw autopkgtest exit code, when the run went through the vm
|
||||
/// escape hatch.
|
||||
pub raw_exit_code: Option<i32>,
|
||||
}
|
||||
|
||||
impl TestRunReport {
|
||||
/// Whether at least one test failed.
|
||||
pub fn has_failures(&self) -> bool {
|
||||
self.outcomes
|
||||
.iter()
|
||||
.any(|outcome| outcome.verdict == Verdict::Fail)
|
||||
}
|
||||
|
||||
/// The pkh exit code: 0 pass/skip/flaky (unless `--fail-on`), 1 test
|
||||
/// failure (or `--fail-on` trigger), 2 for runtime errors (those
|
||||
/// abort the run with an `Err` before a report exists).
|
||||
pub fn exit_code(&self, fail_on_skip: bool, fail_on_no_tests: bool) -> i32 {
|
||||
if self.has_failures() {
|
||||
return 1;
|
||||
}
|
||||
if fail_on_skip
|
||||
&& self
|
||||
.outcomes
|
||||
.iter()
|
||||
.any(|outcome| matches!(outcome.verdict, Verdict::Skip | Verdict::Flaky))
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
if fail_on_no_tests && self.outcomes.is_empty() {
|
||||
return 1;
|
||||
}
|
||||
0
|
||||
}
|
||||
|
||||
/// Human-readable text report: one line per test, then the summary.
|
||||
pub fn render_text(&self) -> String {
|
||||
let mut out = String::new();
|
||||
for outcome in &self.outcomes {
|
||||
let secs = outcome.duration.as_secs_f32();
|
||||
match outcome.verdict {
|
||||
Verdict::Pass => out.push_str(&format!("✓ {} ({secs:.1}s)\n", outcome.name)),
|
||||
Verdict::Fail => {
|
||||
out.push_str(&format!("✗ {}", outcome.name));
|
||||
if let Some(reason) = &outcome.reason {
|
||||
out.push_str(&format!(" — {reason}"));
|
||||
}
|
||||
out.push_str(&format!(" ({secs:.1}s)\n"));
|
||||
}
|
||||
Verdict::Skip => {
|
||||
out.push_str(&format!("- {} (skipped", outcome.name));
|
||||
if let Some(reason) = &outcome.reason {
|
||||
out.push_str(&format!(": {reason}"));
|
||||
}
|
||||
out.push_str(")\n");
|
||||
}
|
||||
Verdict::Flaky => {
|
||||
out.push_str(&format!("~ {} (flaky", outcome.name));
|
||||
if let Some(reason) = &outcome.reason {
|
||||
out.push_str(&format!(": {reason}"));
|
||||
}
|
||||
out.push_str(&format!(") ({secs:.1}s)\n"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let pass = self
|
||||
.outcomes
|
||||
.iter()
|
||||
.filter(|outcome| outcome.verdict == Verdict::Pass)
|
||||
.count();
|
||||
let fail = self
|
||||
.outcomes
|
||||
.iter()
|
||||
.filter(|outcome| outcome.verdict == Verdict::Fail)
|
||||
.count();
|
||||
let skipped = self.outcomes.len() - pass - fail;
|
||||
if self.outcomes.is_empty() {
|
||||
out.push_str("No tests ran.\n");
|
||||
} else if fail > 0 {
|
||||
out.push_str(&format!(
|
||||
"{fail} of {} tests failed{}.\n",
|
||||
self.outcomes.len(),
|
||||
if skipped > 0 {
|
||||
format!(" ({skipped} skipped or flaky)")
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
));
|
||||
} else {
|
||||
out.push_str(&format!(
|
||||
"All {} tests passed{}.\n",
|
||||
self.outcomes.len(),
|
||||
if skipped > 0 {
|
||||
format!(" ({skipped} skipped or flaky)")
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
));
|
||||
}
|
||||
out.push_str(&format!("Artifacts: {}\n", self.artifacts_dir.display()));
|
||||
out
|
||||
}
|
||||
|
||||
/// Schema-versioned JSON report, same content as the text rendering.
|
||||
pub fn render_json(&self) -> String {
|
||||
#[derive(serde::Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
struct Outcome<'a> {
|
||||
name: &'a str,
|
||||
verdict: &'a str,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
reason: Option<&'a String>,
|
||||
duration_secs: f64,
|
||||
}
|
||||
#[derive(serde::Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
struct Report<'a> {
|
||||
schema: &'static str,
|
||||
package: &'a str,
|
||||
version: &'a str,
|
||||
series: &'a str,
|
||||
arch: &'a str,
|
||||
mode: &'a str,
|
||||
binaries: Vec<String>,
|
||||
binaries_note: &'a str,
|
||||
outcomes: Vec<Outcome<'a>>,
|
||||
artifacts_dir: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
raw_exit_code: Option<i32>,
|
||||
}
|
||||
let report = Report {
|
||||
schema: "pkh-test-report-v1",
|
||||
package: &self.package,
|
||||
version: &self.version,
|
||||
series: &self.series,
|
||||
arch: &self.arch,
|
||||
mode: &self.mode,
|
||||
binaries: self
|
||||
.binaries
|
||||
.iter()
|
||||
.map(|path| path.display().to_string())
|
||||
.collect(),
|
||||
binaries_note: &self.binaries_note,
|
||||
outcomes: self
|
||||
.outcomes
|
||||
.iter()
|
||||
.map(|outcome| Outcome {
|
||||
name: &outcome.name,
|
||||
verdict: outcome.verdict.as_str(),
|
||||
reason: outcome.reason.as_ref(),
|
||||
duration_secs: outcome.duration.as_secs_f64(),
|
||||
})
|
||||
.collect(),
|
||||
artifacts_dir: self.artifacts_dir.display().to_string(),
|
||||
raw_exit_code: self.raw_exit_code,
|
||||
};
|
||||
serde_json::to_string_pretty(&report).unwrap_or_else(|_| "{}".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn outcome(name: &str, verdict: Verdict, reason: Option<&str>) -> TestOutcome {
|
||||
TestOutcome {
|
||||
name: name.to_string(),
|
||||
verdict,
|
||||
reason: reason.map(String::from),
|
||||
duration: Duration::from_millis(1500),
|
||||
}
|
||||
}
|
||||
|
||||
fn report(outcomes: Vec<TestOutcome>) -> TestRunReport {
|
||||
TestRunReport {
|
||||
package: "hello".to_string(),
|
||||
version: "2.10-4".to_string(),
|
||||
series: "noble".to_string(),
|
||||
arch: "amd64".to_string(),
|
||||
mode: "chroot".to_string(),
|
||||
binaries: vec![PathBuf::from("../hello_2.10-4_amd64.deb")],
|
||||
binaries_note: "fresh pkh deb output".to_string(),
|
||||
outcomes,
|
||||
artifacts_dir: PathBuf::from("/var/tmp/pkh/tests/hello-noble-amd64-x/"),
|
||||
raw_exit_code: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The exit-code contract: failures dominate, `--fail-on` promotes
|
||||
/// skips and empty runs.
|
||||
#[test]
|
||||
fn exit_code_matrix() {
|
||||
let pass = report(vec![outcome("a", Verdict::Pass, None)]);
|
||||
assert_eq!(pass.exit_code(false, false), 0);
|
||||
assert_eq!(pass.exit_code(true, true), 0);
|
||||
|
||||
let skip = report(vec![
|
||||
outcome("a", Verdict::Pass, None),
|
||||
outcome("b", Verdict::Skip, Some("needs root")),
|
||||
]);
|
||||
assert_eq!(skip.exit_code(false, false), 0);
|
||||
assert_eq!(skip.exit_code(true, false), 1);
|
||||
|
||||
let fail = report(vec![
|
||||
outcome("a", Verdict::Pass, None),
|
||||
outcome("b", Verdict::Fail, Some("exit code 1")),
|
||||
]);
|
||||
assert_eq!(fail.exit_code(false, false), 1);
|
||||
// A failure dominates every other trigger.
|
||||
assert_eq!(fail.exit_code(true, true), 1);
|
||||
|
||||
let empty = report(vec![]);
|
||||
assert_eq!(empty.exit_code(false, false), 0);
|
||||
assert_eq!(empty.exit_code(false, true), 1);
|
||||
|
||||
let flaky = report(vec![outcome("a", Verdict::Flaky, Some("passed on retry"))]);
|
||||
assert_eq!(flaky.exit_code(false, false), 0);
|
||||
assert_eq!(flaky.exit_code(true, false), 1);
|
||||
}
|
||||
|
||||
/// The text rendering shows every verdict with reasons and a summary.
|
||||
#[test]
|
||||
fn text_rendering() {
|
||||
let run = report(vec![
|
||||
outcome("smoke", Verdict::Pass, None),
|
||||
outcome("l10n", Verdict::Fail, Some("exit code 1")),
|
||||
outcome("heavy", Verdict::Skip, Some("requires a VM")),
|
||||
]);
|
||||
let text = run.render_text();
|
||||
assert!(text.contains("✓ smoke"), "{text}");
|
||||
assert!(text.contains("✗ l10n — exit code 1"), "{text}");
|
||||
assert!(text.contains("- heavy (skipped: requires a VM)"), "{text}");
|
||||
assert!(
|
||||
text.contains("1 of 3 tests failed (1 skipped or flaky)"),
|
||||
"{text}"
|
||||
);
|
||||
assert!(
|
||||
text.contains("Artifacts: /var/tmp/pkh/tests/hello-noble-amd64-x/"),
|
||||
"{text}"
|
||||
);
|
||||
|
||||
let empty = report(vec![]).render_text();
|
||||
assert!(empty.contains("No tests ran."), "{empty}");
|
||||
|
||||
let clean = report(vec![outcome("smoke", Verdict::Pass, None)]).render_text();
|
||||
assert!(clean.contains("All 1 tests passed."), "{clean}");
|
||||
}
|
||||
|
||||
/// The JSON report round-trips the report shape with a schema marker.
|
||||
#[test]
|
||||
fn json_rendering() {
|
||||
let run = report(vec![outcome("smoke", Verdict::Pass, None)]);
|
||||
let json: serde_json::Value = serde_json::from_str(&run.render_json()).unwrap();
|
||||
assert_eq!(json["schema"], "pkh-test-report-v1");
|
||||
assert_eq!(json["package"], "hello");
|
||||
assert_eq!(json["mode"], "chroot");
|
||||
assert_eq!(json["outcomes"][0]["verdict"], "pass");
|
||||
assert_eq!(json["outcomes"][0]["duration_secs"], 1.5);
|
||||
assert!(json.get("reason").is_none() && json["outcomes"][0].get("reason").is_none());
|
||||
assert!(json.get("raw_exit_code").is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,544 @@
|
||||
//! The DEP-8 per-test contract: restriction gates, dependency
|
||||
//! installation, the run environment, verdicts and artifact retrieval.
|
||||
//!
|
||||
//! The contract (autopkgtest 6.x `README.package-tests`): tests run in
|
||||
//! the unpacked source tree with a fresh `$AUTOPKGTEST_TMP`; exit 0 with
|
||||
//! empty stderr passes, any stderr output fails unless the
|
||||
//! `allow-stderr` restriction is declared, exit 77 skips under
|
||||
//! `skippable`, and a per-test timeout kills the test.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::context::{Context, LineSink, Stream};
|
||||
use crate::debian::control::ControlInfo;
|
||||
use crate::debian::tests_control::Test;
|
||||
use crate::logfmt::GenericClassifier;
|
||||
use crate::report::BuildView;
|
||||
use crate::test::expand::expand_depends;
|
||||
use crate::test::outcome::{TestOutcome, Verdict};
|
||||
|
||||
/// Grace period `timeout --kill-after` gives a test after SIGTERM.
|
||||
const KILL_AFTER_SECS: u64 = 15;
|
||||
|
||||
/// Everything one test run needs from the orchestration.
|
||||
pub(crate) struct RunContext<'a> {
|
||||
/// The testbed context.
|
||||
pub ctx: Arc<Context>,
|
||||
/// Context-side path of the staged source tree.
|
||||
pub tree: String,
|
||||
/// Context-side scratch root (temporary directories, artifacts).
|
||||
pub staging_root: String,
|
||||
/// The testbed architecture.
|
||||
pub arch: String,
|
||||
/// The host architecture (for `skip-foreign-architecture`).
|
||||
pub host_arch: String,
|
||||
/// Whether root-requiring tests can run.
|
||||
pub supports_root: bool,
|
||||
/// Per-test timeout in seconds.
|
||||
pub timeout_secs: u64,
|
||||
/// The view's tee sink, when the view consumes subprocess lines.
|
||||
pub sink: Option<Arc<dyn LineSink>>,
|
||||
/// Where run events are reported.
|
||||
pub view: &'a dyn BuildView,
|
||||
/// The source control file, for `Depends:` expansion.
|
||||
pub control: ControlInfo,
|
||||
/// The binary package names built for the tests.
|
||||
pub binaries: Vec<String>,
|
||||
/// Base environment for testbed commands.
|
||||
pub env: HashMap<String, String>,
|
||||
/// Host directory holding the run's artifacts.
|
||||
pub artifacts_dir: PathBuf,
|
||||
}
|
||||
|
||||
/// Run the tests in declaration order, filtered by the caller's
|
||||
/// selection, and return one outcome per selected test.
|
||||
pub(crate) fn run_tests(
|
||||
run: &RunContext<'_>,
|
||||
tests: &[Test],
|
||||
only: &[String],
|
||||
skipped: &[String],
|
||||
) -> Vec<TestOutcome> {
|
||||
let selected: Vec<&Test> = tests
|
||||
.iter()
|
||||
.filter(|test| only.is_empty() || only.iter().any(|name| name == &test.name))
|
||||
.collect();
|
||||
let total = selected.len();
|
||||
|
||||
run.view
|
||||
.phase("Running tests", Box::new(GenericClassifier::new()));
|
||||
let mut outcomes = Vec::new();
|
||||
for (index, test) in selected.into_iter().enumerate() {
|
||||
let outcome = if skipped.iter().any(|name| name == &test.name) {
|
||||
TestOutcome {
|
||||
name: test.name.clone(),
|
||||
verdict: Verdict::Skip,
|
||||
reason: Some("excluded by --skip-test".to_string()),
|
||||
duration: Duration::ZERO,
|
||||
}
|
||||
} else {
|
||||
run_one(run, test)
|
||||
};
|
||||
run.view.progress("Tests", index + 1, total);
|
||||
outcomes.push(outcome);
|
||||
}
|
||||
outcomes
|
||||
}
|
||||
|
||||
/// The result of one raw test execution, as seen by the verdict logic.
|
||||
pub(crate) struct RawRun {
|
||||
/// Process exit code (`None` when killed by a signal).
|
||||
pub code: Option<i32>,
|
||||
/// Whether the test produced any stderr output.
|
||||
pub stderr_nonempty: bool,
|
||||
/// Whether the `timeout` wrapper killed the test.
|
||||
pub timed_out: bool,
|
||||
}
|
||||
|
||||
/// The verdict for one raw run, per the DEP-8 contract.
|
||||
pub(crate) fn decide(test: &Test, raw: &RawRun, timeout_secs: u64) -> (Verdict, Option<String>) {
|
||||
if raw.timed_out {
|
||||
return (
|
||||
Verdict::Fail,
|
||||
Some(format!("timed out after {timeout_secs}s")),
|
||||
);
|
||||
}
|
||||
match raw.code {
|
||||
// `timeout` reports its kill with 124; a test exiting 124 by
|
||||
// itself is indistinguishable and counted as a timeout.
|
||||
Some(124) => (
|
||||
Verdict::Fail,
|
||||
Some(format!("timed out after {timeout_secs}s")),
|
||||
),
|
||||
None => (Verdict::Fail, Some("killed by a signal".to_string())),
|
||||
// A self-skipping test may be chatty: the skip verdict comes
|
||||
// before the stderr rule.
|
||||
Some(77) if test.has_restriction("skippable") => (
|
||||
Verdict::Skip,
|
||||
Some("the test skipped itself (exit 77)".to_string()),
|
||||
),
|
||||
Some(0) if !raw.stderr_nonempty || test.has_restriction("allow-stderr") => {
|
||||
(Verdict::Pass, None)
|
||||
}
|
||||
Some(0) => (Verdict::Fail, Some("the test wrote to stderr".to_string())),
|
||||
Some(code) => (Verdict::Fail, Some(format!("exit code {code}"))),
|
||||
}
|
||||
}
|
||||
|
||||
/// The static reason a test cannot run on this testbed, if any.
|
||||
pub(crate) fn restriction_skip(
|
||||
test: &Test,
|
||||
supports_root: bool,
|
||||
foreign_arch: bool,
|
||||
) -> Option<String> {
|
||||
if test.has_restriction("isolation-machine") {
|
||||
return Some("requires a virtual machine (isolation-machine); use --mode vm".to_string());
|
||||
}
|
||||
if test.has_restriction("needs-reboot") {
|
||||
return Some("requires reboot support (needs-reboot); use --mode vm".to_string());
|
||||
}
|
||||
if test.has_restriction("isolation-container") {
|
||||
return Some("requires container isolation (isolation-container)".to_string());
|
||||
}
|
||||
if test.has_restriction("skip-foreign-architecture") && foreign_arch {
|
||||
return Some("skipped on a foreign architecture (skip-foreign-architecture)".to_string());
|
||||
}
|
||||
if (test.has_restriction("needs-root") || test.has_restriction("needs-sudo")) && !supports_root
|
||||
{
|
||||
return Some("requires root on this testbed".to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Run one test through the DEP-8 contract.
|
||||
fn run_one(run: &RunContext<'_>, test: &Test) -> TestOutcome {
|
||||
let started = Instant::now();
|
||||
if let Some(reason) = restriction_skip(test, run.supports_root, run.arch != run.host_arch) {
|
||||
return TestOutcome {
|
||||
name: test.name.clone(),
|
||||
verdict: Verdict::Skip,
|
||||
reason: Some(reason),
|
||||
duration: Duration::ZERO,
|
||||
};
|
||||
}
|
||||
|
||||
// Test dependencies: expanded per stanza, installed before the test.
|
||||
if let Err(reason) = install_test_deps(run, test) {
|
||||
return outcome_fail(test, started, reason);
|
||||
}
|
||||
|
||||
// build-needed: the tests need the tree in its built state.
|
||||
if test.has_restriction("build-needed")
|
||||
&& let Err(reason) = build_tree(run)
|
||||
{
|
||||
return outcome_fail(test, started, reason);
|
||||
}
|
||||
|
||||
let (raw, duration) = execute(run, test);
|
||||
let (mut verdict, mut reason) = decide(test, &raw, run.timeout_secs);
|
||||
|
||||
// flaky: one tolerated failure is retried once and reported as FLAKY
|
||||
// rather than FAIL (autopkgtest's semantics).
|
||||
if verdict == Verdict::Fail && test.has_restriction("flaky") {
|
||||
let (retry_raw, _) = execute(run, test);
|
||||
let (retry_verdict, retry_reason) = decide(test, &retry_raw, run.timeout_secs);
|
||||
reason = if retry_verdict == Verdict::Pass {
|
||||
Some("passed on retry".to_string())
|
||||
} else {
|
||||
reason.or(retry_reason)
|
||||
};
|
||||
verdict = Verdict::Flaky;
|
||||
}
|
||||
|
||||
TestOutcome {
|
||||
name: test.name.clone(),
|
||||
verdict,
|
||||
reason,
|
||||
duration,
|
||||
}
|
||||
}
|
||||
|
||||
fn outcome_fail(test: &Test, started: Instant, reason: String) -> TestOutcome {
|
||||
TestOutcome {
|
||||
name: test.name.clone(),
|
||||
verdict: Verdict::Fail,
|
||||
reason: Some(reason),
|
||||
duration: started.elapsed(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute a test once, returning the raw result and its duration.
|
||||
fn execute(run: &RunContext<'_>, test: &Test) -> (RawRun, Duration) {
|
||||
let started = Instant::now();
|
||||
|
||||
// Fresh per-test scratch and artifact directories.
|
||||
let tmp_dir = run
|
||||
.ctx
|
||||
.create_temp_dir()
|
||||
.unwrap_or_else(|_| format!("{}/tmp-{}", run.staging_root, safe_name(&test.name)));
|
||||
let artifacts_dir = format!("{}/artifacts/{}", run.staging_root, safe_name(&test.name));
|
||||
let _ = run
|
||||
.ctx
|
||||
.command("mkdir")
|
||||
.arg("-p")
|
||||
.arg(&artifacts_dir)
|
||||
.status();
|
||||
|
||||
// The test's argv, wrapped in the per-test timeout.
|
||||
let mut argv: Vec<String> = vec![
|
||||
"timeout".to_string(),
|
||||
"--foreground".to_string(),
|
||||
format!("--kill-after={KILL_AFTER_SECS}"),
|
||||
run.timeout_secs.to_string(),
|
||||
];
|
||||
match (&test.script, &test.command) {
|
||||
(Some(script), _) => {
|
||||
let path = test_script_path(run, test, script);
|
||||
// Scripts may be shipped without the executable bit.
|
||||
let _ = run.ctx.command("chmod").arg("+x").arg(&path).status();
|
||||
argv.push(path);
|
||||
}
|
||||
(None, Some(command)) => {
|
||||
argv.push("bash".to_string());
|
||||
argv.push("-e".to_string());
|
||||
argv.push("-c".to_string());
|
||||
argv.push(command.clone());
|
||||
}
|
||||
(None, None) => unreachable!("parser rejects stanzas with neither Tests nor Test-Command"),
|
||||
}
|
||||
|
||||
// The DEP-8 environment. AUTOPKGTEST_* are the current names; the
|
||||
// ADT_* aliases are still read by older tests.
|
||||
let mut env = run.env.clone();
|
||||
for (key, value) in [
|
||||
("AUTOPKGTEST_TMP", tmp_dir.as_str()),
|
||||
("ADTTMP", tmp_dir.as_str()),
|
||||
("AUTOPKGTEST_ARTIFACTS", artifacts_dir.as_str()),
|
||||
("ADT_ARTIFACTS", artifacts_dir.as_str()),
|
||||
("AUTOPKGTEST_TESTBED_ARCH", run.arch.as_str()),
|
||||
] {
|
||||
env.insert(key.to_string(), value.to_string());
|
||||
}
|
||||
|
||||
let capture = Arc::new(TestSink::new(
|
||||
host_artifacts_dir(run, &test.name),
|
||||
run.sink.clone(),
|
||||
));
|
||||
let mut cmd = run.ctx.command(&argv[0]);
|
||||
cmd.args(&argv[1..])
|
||||
.envs(env.iter())
|
||||
.current_dir(&run.tree)
|
||||
.capture(capture.clone());
|
||||
let status = cmd.status().ok();
|
||||
|
||||
// The scratch directory is disposable; the artifacts directory was
|
||||
// already retrieved by the sink's host-side files plus the copy
|
||||
// below (tests may have written into $AUTOPKGTEST_ARTIFACTS).
|
||||
let _ = run.ctx.command("rm").arg("-rf").arg(&tmp_dir).status();
|
||||
retrieve_artifacts(run, &test.name);
|
||||
|
||||
let raw = RawRun {
|
||||
code: status.as_ref().and_then(|status| status.code()),
|
||||
stderr_nonempty: capture.stderr_lines() > 0,
|
||||
timed_out: status.as_ref().and_then(|status| status.code()) == Some(124),
|
||||
};
|
||||
(raw, started.elapsed())
|
||||
}
|
||||
|
||||
/// Install a test stanza's expanded dependencies.
|
||||
fn install_test_deps(run: &RunContext<'_>, test: &Test) -> Result<(), String> {
|
||||
let specs = expand_depends(
|
||||
test.depends.as_deref(),
|
||||
&run.control,
|
||||
&run.binaries,
|
||||
&run.arch,
|
||||
)
|
||||
.map_err(|e| format!("cannot expand the test dependencies: {e}"))?;
|
||||
if specs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
run.view
|
||||
.message(&format!("Installing dependencies of {}", test.name));
|
||||
let mut cmd = run.ctx.command("apt-get");
|
||||
cmd.envs(run.env.clone())
|
||||
.arg("-y")
|
||||
.arg("--no-install-recommends")
|
||||
.arg("install");
|
||||
for spec in &specs {
|
||||
cmd.arg(spec);
|
||||
}
|
||||
let status = cmd.status().map_err(|e| e.to_string())?;
|
||||
if status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!(
|
||||
"dependency installation failed ({})",
|
||||
specs.join(", ")
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Bring the staged tree to its built state (`build-needed` tests): the
|
||||
/// autopkgtest build, with the nocheck profile — the tests need the
|
||||
/// build products, not another set of binaries.
|
||||
fn build_tree(run: &RunContext<'_>) -> Result<(), String> {
|
||||
let mut env = run.env.clone();
|
||||
env.insert("DEB_BUILD_OPTIONS".to_string(), "nocheck".to_string());
|
||||
let mut cmd = run.ctx.command("dpkg-buildpackage");
|
||||
cmd.envs(env.iter())
|
||||
.arg("-us")
|
||||
.arg("-uc")
|
||||
.arg("-b")
|
||||
.current_dir(&run.tree);
|
||||
let status = cmd.status().map_err(|e| e.to_string())?;
|
||||
if status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err("building the source tree failed (build-needed)".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// The context-side path of a test script: `Tests-Directory:` replaces
|
||||
/// the `debian/tests` path segment.
|
||||
fn test_script_path(run: &RunContext<'_>, test: &Test, script: &str) -> String {
|
||||
let base = test.tests_directory.as_deref().unwrap_or("debian/tests");
|
||||
format!("{}/{base}/{script}", run.tree)
|
||||
}
|
||||
|
||||
/// Retrieve a test's artifacts directory from the testbed into the run's
|
||||
/// host directory (best-effort: tests may not have written anything).
|
||||
fn retrieve_artifacts(run: &RunContext<'_>, test_name: &str) {
|
||||
let remote = PathBuf::from(format!(
|
||||
"{}/artifacts/{}",
|
||||
run.staging_root,
|
||||
safe_name(test_name)
|
||||
));
|
||||
if !run.ctx.exists(&remote).unwrap_or(false) {
|
||||
return;
|
||||
}
|
||||
let host = host_artifacts_dir(run, test_name);
|
||||
let _ = fs::create_dir_all(&host);
|
||||
let _ = run.ctx.retrieve_path(&remote, &host);
|
||||
}
|
||||
|
||||
/// The host-side artifacts directory of one test.
|
||||
fn host_artifacts_dir(run: &RunContext<'_>, test_name: &str) -> PathBuf {
|
||||
run.artifacts_dir
|
||||
.join("artifacts")
|
||||
.join(safe_name(test_name))
|
||||
}
|
||||
|
||||
/// Filesystem-safe directory name for a test name (script paths may
|
||||
/// contain slashes).
|
||||
fn safe_name(name: &str) -> String {
|
||||
name.replace('/', "_")
|
||||
}
|
||||
|
||||
/// Per-test capture: appends stdout/stderr lines to files in the run's
|
||||
/// artifacts directory, counts stderr output (the DEP-8 failure rule)
|
||||
/// and forwards every line to the view's tee sink.
|
||||
struct TestSink {
|
||||
stdout: Mutex<fs::File>,
|
||||
stderr: Mutex<fs::File>,
|
||||
stderr_lines: AtomicUsize,
|
||||
forward: Option<Arc<dyn LineSink>>,
|
||||
}
|
||||
|
||||
impl TestSink {
|
||||
/// Build the sink; the log files live under the test's artifacts
|
||||
/// directory on the host.
|
||||
fn new(host_dir: PathBuf, forward: Option<Arc<dyn LineSink>>) -> TestSink {
|
||||
let _ = fs::create_dir_all(&host_dir);
|
||||
let open = |name: &str| {
|
||||
fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(host_dir.join(name))
|
||||
.unwrap_or_else(|_| {
|
||||
// No artifacts directory: capture into a null file.
|
||||
fs::File::create("/dev/null").expect("/dev/null is always openable")
|
||||
})
|
||||
};
|
||||
TestSink {
|
||||
stdout: Mutex::new(open("stdout.log")),
|
||||
stderr: Mutex::new(open("stderr.log")),
|
||||
stderr_lines: AtomicUsize::new(0),
|
||||
forward,
|
||||
}
|
||||
}
|
||||
|
||||
/// Number of stderr lines seen.
|
||||
fn stderr_lines(&self) -> usize {
|
||||
self.stderr_lines.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
impl LineSink for TestSink {
|
||||
fn line(&self, stream: Stream, line: &str) {
|
||||
let mut file = match stream {
|
||||
Stream::Stdout => self
|
||||
.stdout
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner),
|
||||
Stream::Stderr => {
|
||||
self.stderr_lines.fetch_add(1, Ordering::Relaxed);
|
||||
self.stderr
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
};
|
||||
let _ = writeln!(file, "{line}");
|
||||
if let Some(forward) = &self.forward {
|
||||
forward.line(stream, line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn test(restrictions: &[&str]) -> Test {
|
||||
Test {
|
||||
name: "smoke".to_string(),
|
||||
script: Some("smoke".to_string()),
|
||||
command: None,
|
||||
depends: None,
|
||||
restrictions: restrictions.iter().map(|s| s.to_string()).collect(),
|
||||
tests_directory: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn raw(code: Option<i32>, stderr: bool) -> RawRun {
|
||||
RawRun {
|
||||
code,
|
||||
stderr_nonempty: stderr,
|
||||
timed_out: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The verdict matrix: exit/stderr/skip/timeout rules.
|
||||
#[test]
|
||||
fn verdict_matrix() {
|
||||
let plain = test(&[]);
|
||||
assert_eq!(
|
||||
decide(&plain, &raw(Some(0), false), 10),
|
||||
(Verdict::Pass, None)
|
||||
);
|
||||
assert_eq!(
|
||||
decide(&plain, &raw(Some(0), true), 10),
|
||||
(Verdict::Fail, Some("the test wrote to stderr".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
decide(&plain, &raw(Some(1), false), 10),
|
||||
(Verdict::Fail, Some("exit code 1".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
decide(&plain, &raw(None, false), 10),
|
||||
(Verdict::Fail, Some("killed by a signal".to_string()))
|
||||
);
|
||||
assert_eq!(
|
||||
decide(&plain, &raw(Some(124), false), 42),
|
||||
(Verdict::Fail, Some("timed out after 42s".to_string()))
|
||||
);
|
||||
|
||||
// exit 77 skips only under `skippable` — and a chatty
|
||||
// self-skipping test still skips (the verdict comes first).
|
||||
assert_eq!(decide(&plain, &raw(Some(77), false), 10).0, Verdict::Fail);
|
||||
let skippable = test(&["skippable"]);
|
||||
assert_eq!(
|
||||
decide(&skippable, &raw(Some(77), true), 10).0,
|
||||
Verdict::Skip
|
||||
);
|
||||
|
||||
// allow-stderr lifts the stderr rule.
|
||||
let chatty = test(&["allow-stderr"]);
|
||||
assert_eq!(decide(&chatty, &raw(Some(0), true), 10).0, Verdict::Pass);
|
||||
}
|
||||
|
||||
/// Restriction gates: machine isolation and root requirements skip
|
||||
/// with an actionable reason.
|
||||
#[test]
|
||||
fn restriction_gates() {
|
||||
let plain = test(&[]);
|
||||
assert_eq!(restriction_skip(&plain, true, false), None);
|
||||
assert_eq!(restriction_skip(&plain, false, true), None);
|
||||
|
||||
let vm_only = test(&["isolation-machine"]);
|
||||
assert!(
|
||||
restriction_skip(&vm_only, true, false)
|
||||
.unwrap()
|
||||
.contains("--mode vm")
|
||||
);
|
||||
|
||||
let reboot = test(&["needs-reboot"]);
|
||||
assert!(restriction_skip(&reboot, true, false).is_some());
|
||||
|
||||
let container = test(&["isolation-container"]);
|
||||
assert!(restriction_skip(&container, true, false).is_some());
|
||||
|
||||
let root = test(&["needs-root"]);
|
||||
assert!(restriction_skip(&root, false, false).is_some());
|
||||
assert_eq!(restriction_skip(&root, true, false), None);
|
||||
|
||||
let sudo = test(&["needs-sudo"]);
|
||||
assert!(restriction_skip(&sudo, false, false).is_some());
|
||||
|
||||
let native_only = test(&["skip-foreign-architecture"]);
|
||||
assert!(restriction_skip(&native_only, true, true).is_some());
|
||||
assert_eq!(restriction_skip(&native_only, true, false), None);
|
||||
}
|
||||
|
||||
/// Test names map to filesystem-safe artifact directory names.
|
||||
#[test]
|
||||
fn names_are_sanitized() {
|
||||
assert_eq!(safe_name("checks/cli"), "checks_cli");
|
||||
assert_eq!(safe_name("smoke"), "smoke");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user