diff --git a/src/main.rs b/src/main.rs index cca1631..af45059 100644 --- a/src/main.rs +++ b/src/main.rs @@ -391,7 +391,10 @@ fn main() { .arg(arg!(-p --pocket "Resolve test dependencies from a specific distribution pocket (updates, security, proposed)").required(false)) .arg(arg!(--ppa "Add a PPA as test dependency source (can be specified multiple times)").required(false).action(clap::ArgAction::Append)) .arg(arg!(--inject "Inject a package into the testbed before the tests (.deb path or package name, can be specified multiple times)").required(false).action(clap::ArgAction::Append)) - .arg(arg!(--mode "Testbed mode: chroot (default, an ephemeral unshare chroot) or local (the current context, no isolation)").required(false)) + .arg(arg!(--mode "Testbed mode: chroot (default, an ephemeral unshare chroot), local (the current context, no isolation) or vm (a qemu VM through autopkgtest)").required(false)) + .arg(arg!(--image "VM image for --mode vm (required there)").required(false)) + .arg(arg!(--cpus "VM virtual CPUs (vm mode)").required(false).value_parser(clap::value_parser!(u32))) + .arg(arg!(--"ram-size" "VM memory size in MiB (vm mode)").required(false).value_parser(clap::value_parser!(u32))) .arg(arg!(--debs "Explicit binary package to test, overriding the pkh deb output reuse (can be specified multiple times)").required(false).action(clap::ArgAction::Append)) .arg( clap::Arg::new("no_build") @@ -1062,6 +1065,13 @@ fn main() { .map(|v| v.cloned().collect()) .unwrap_or_default(), mode, + image: sub_matches + .get_one::("image") + .map(std::path::PathBuf::from), + cpus: sub_matches.get_one::("cpus").copied(), + ram_size: sub_matches + .get_one::("ram-size") + .map(|v| v.to_string()), debs: sub_matches .get_many::("debs") .map(|v| v.map(std::path::PathBuf::from).collect()) diff --git a/src/test/bed.rs b/src/test/bed.rs index 8130550..ce73ce5 100644 --- a/src/test/bed.rs +++ b/src/test/bed.rs @@ -48,6 +48,8 @@ impl Bed { Ok(Bed::Chroot(guard)) } TestMode::Local => Ok(Bed::Local(base_ctx)), + // The vm mode never builds a Bed: run_vm execs autopkgtest. + TestMode::Vm => unreachable!("vm mode does not prepare a Bed"), } } diff --git a/src/test/mod.rs b/src/test/mod.rs index 261342c..1fade65 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -17,6 +17,7 @@ pub mod expand; pub mod options; pub mod outcome; pub(crate) mod runner; +pub(crate) mod vm; pub use options::{TestMode, TestOptions}; pub use outcome::TestRunReport; @@ -117,6 +118,38 @@ pub async fn run(opts: TestOptions<'_>) -> Result) -> Result Result> { if report.has_failures() { view.finish_failure(); } else { - view.finish_success(&[run_dir]); + view.finish_success(std::slice::from_ref(&report.artifacts_dir)); } Ok(report) } @@ -535,6 +578,12 @@ fn create_run_dir(package: &str, series: &str, arch: &str) -> std::io::Result PathBuf { + let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%S"); + runs_root().join(format!("{package}-{series}-{arch}-{timestamp}")) +} + /// The test-run artifacts root: `$PKH_TESTS_DIR`, defaulting to /// `/var/tmp/pkh/tests`. pub fn runs_root() -> PathBuf { diff --git a/src/test/options.rs b/src/test/options.rs index edea088..e1cf8f0 100644 --- a/src/test/options.rs +++ b/src/test/options.rs @@ -17,6 +17,9 @@ pub enum TestMode { Chroot, /// The current context directly: no isolation, tests can rewrite it. Local, + /// A qemu virtual machine, through the installed autopkgtest's qemu + /// runner — the one backend pkh does not reimplement. + Vm, } impl TestMode { @@ -25,6 +28,7 @@ impl TestMode { match self { TestMode::Chroot => "chroot", TestMode::Local => "local", + TestMode::Vm => "vm", } } @@ -33,6 +37,7 @@ impl TestMode { match value { "chroot" => Some(TestMode::Chroot), "local" => Some(TestMode::Local), + "vm" => Some(TestMode::Vm), _ => None, } } @@ -58,6 +63,12 @@ pub struct TestOptions<'a> { pub inject: Vec, /// Testbed mode; defaults to [`TestMode::Chroot`]. pub mode: Option, + /// VM image for [`TestMode::Vm`] (required there). + pub image: Option, + /// VM virtual CPUs ([`TestMode::Vm`], passthrough). + pub cpus: Option, + /// VM memory size in MiB ([`TestMode::Vm`], passthrough). + pub ram_size: Option, /// Explicit binary packages to test; overrides the `pkh deb` output /// reuse. pub debs: Vec, @@ -106,6 +117,9 @@ impl Default for TestOptions<'_> { ppa: Vec::new(), inject: Vec::new(), mode: None, + image: None, + cpus: None, + ram_size: None, debs: Vec::new(), no_build: false, build: false, @@ -197,8 +211,9 @@ mod tests { fn mode_parsing() { assert_eq!(TestMode::parse("chroot"), Some(TestMode::Chroot)); assert_eq!(TestMode::parse("local"), Some(TestMode::Local)); - assert_eq!(TestMode::parse("vm"), None); + assert_eq!(TestMode::parse("vm"), Some(TestMode::Vm)); + assert_eq!(TestMode::parse("other"), None); assert_eq!(TestMode::default(), TestMode::Chroot); - assert_eq!(TestMode::Local.label(), "local"); + assert_eq!(TestMode::Vm.label(), "vm"); } } diff --git a/src/test/vm.rs b/src/test/vm.rs new file mode 100644 index 0000000..11c6321 --- /dev/null +++ b/src/test/vm.rs @@ -0,0 +1,242 @@ +//! The `--mode vm` escape hatch: exec the installed `autopkgtest` with +//! its qemu runner. +//! +//! VM virtualization is the one backend pkh deliberately does not +//! reimplement: autopkgtest-virt-qemu carries a serial-console control +//! protocol, image overlay management and the reboot protocol, and it is +//! the only runner satisfying `isolation-machine`/`needs-reboot` tests. +//! Everything around it stays pkh's: binary resolution, the report +//! model, the exit-code contract and the artifacts directory. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use crate::debian::tests_control::Test; +use crate::interrupt; +use crate::report::BuildView; +use crate::test::outcome::{TestOutcome, Verdict}; + +/// Run the tests through autopkgtest's qemu runner. +/// +/// `tree` is the source tree, `binaries` the fresh binary packages under +/// test (may be empty: autopkgtest then builds inside the VM), +/// `run_dir` the artifacts directory (passed as `-o`; autopkgtest +/// requires it to not exist yet) and `image` the VM image path. Returns +/// the per-test outcomes and autopkgtest's raw exit code; the caller +/// builds the full report around them. +pub(crate) fn run_vm( + tree: &Path, + binaries: &[PathBuf], + tests: &[Test], + run_dir: &Path, + image: Option<&Path>, + opts: &crate::test::options::TestOptions<'_>, + view: &dyn BuildView, +) -> Result<(Vec, i32), Box> { + let Some(image) = image else { + return Err( + "--mode vm needs a VM image: pass --image, or build one with \ + 'autopkgtest-build-qemu ' (Debian) or \ + 'autopkgtest-buildvm-ubuntu-cloud -r -a ' (Ubuntu)" + .into(), + ); + }; + if !image.exists() { + return Err(format!( + "VM image '{}' does not exist: pass --image, or build one with \ + 'autopkgtest-build-qemu ' (Debian) or \ + 'autopkgtest-buildvm-ubuntu-cloud -r -a ' (Ubuntu)", + image.display() + ) + .into()); + } + if !opts.ppa.is_empty() || opts.pocket.is_some() || !opts.inject.is_empty() { + return Err( + "--pocket, --ppa and --inject are not supported with --mode vm yet: \ + configure them inside the image or use a chroot/local run" + .into(), + ); + } + + view.message("Running autopkgtest with the qemu runner"); + + let mut command = Command::new("autopkgtest"); + command.arg("-o").arg(run_dir); + let summary_path = run_dir.join("autopkgtest-summary"); + command.arg("--summary").arg(&summary_path); + if opts.apt_upgrade { + command.arg("--apt-upgrade"); + } + for name in &opts.test_names { + command.arg("--test-name").arg(name); + } + for name in &opts.skip_tests { + command.arg("--skip-test").arg(name); + } + for setup in &opts.setup_commands { + command.arg("--setup-commands").arg(setup); + } + if opts.shell { + command.arg("--shell"); + } + if opts.shell_fail { + command.arg("--shell-fail"); + } + if let Some(secs) = opts.timeout { + command.arg("--timeout").arg(secs.to_string()); + } + if let Some(factor) = opts.timeout_factor { + command.arg("--timeout-factor").arg(factor.to_string()); + } + for deb in binaries { + // Absolute paths: autopkgtest reads a bare name as an archive + // source package. + command.arg(deb.canonicalize()?); + } + command.arg(tree.canonicalize()?); + command.arg("--"); + command.arg("qemu"); + if let Some(cpus) = opts.cpus { + command.arg("--cpus").arg(cpus.to_string()); + } + if let Some(ram) = &opts.ram_size { + command.arg("--ram-size").arg(ram); + } + command.arg(image); + + // The autopkgtest process must die with us: run it in its own + // process group and SIGTERM the group from the interrupt-time + // cleanup hook (autopkgtest cleans up its testbed on TERM). + let mut child = command + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn() + .map_err(|e| format!("cannot run autopkgtest (is it installed?): {e}"))?; + let pid = child.id(); + // Best-effort: without its own group, the cleanup kill below would + // only reach the autopkgtest front-end, not its virt server. + unsafe { libc::setpgid(pid as libc::pid_t, pid as libc::pid_t) }; + let hook = interrupt::register_cleanup_hook(Box::new(move || { + // SAFETY: kill(2) to the spawned autopkgtest process group + unsafe { libc::kill(-(pid as libc::pid_t), libc::SIGTERM) }; + })); + + let status = child.wait()?; + drop(hook); + let raw_code = status.code().unwrap_or(16); + + // autopkgtest's own failure classes are runtime errors here; the + // pass/fail/skip classes become the report. + if matches!(raw_code, 12 | 14 | 16 | 20) { + return Err(format!( + "autopkgtest failed (exit {raw_code}): check the artifacts in '{}'", + run_dir.display() + ) + .into()); + } + + let outcomes = parse_summary(&summary_path, tests)?; + Ok((outcomes, raw_code)) +} + +/// Parse autopkgtest's `--summary` file: one line per test, ` +/// PASS|FAIL|SKIP|FLAKY [reason...]`. Tolerant by design — the format is +/// stable but not formally versioned; unparseable lines are skipped, and +/// a missing file yields no outcomes (the raw exit code carries the +/// verdict). +pub(crate) fn parse_summary( + summary_path: &Path, + tests: &[Test], +) -> Result, Box> { + let mut outcomes = Vec::new(); + let Ok(content) = std::fs::read_to_string(summary_path) else { + return Ok(outcomes); + }; + for line in content.lines() { + let mut tokens = line.splitn(3, ' '); + let Some(name) = tokens.next() else { + continue; + }; + let Some(verdict) = tokens.next() else { + continue; + }; + // Summary names carry the argid prefix ("-"); a + // declared test matching by suffix wins, else the token stands. + let name = name.to_string(); + let name = match tests + .iter() + .find(|test| name.ends_with(&test.name) || test.name == name) + { + Some(test) => test.name.clone(), + None => name, + }; + let reason = tokens.next().map(str::to_string); + let verdict = match verdict.trim() { + "PASS" => Verdict::Pass, + "FAIL" => Verdict::Fail, + "SKIP" => Verdict::Skip, + "FLAKY" => Verdict::Flaky, + _ => continue, + }; + outcomes.push(TestOutcome { + name, + verdict, + reason, + duration: std::time::Duration::ZERO, + }); + } + Ok(outcomes) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::debian::tests_control::Test; + + fn test(name: &str) -> Test { + Test { + name: name.to_string(), + script: Some(name.to_string()), + command: None, + depends: None, + restrictions: Vec::new(), + tests_directory: None, + } + } + + /// The summary parser maps the four verdicts, keeps reasons and + /// resolves argid-prefixed names back to the declared tests. + #[test] + fn summary_parsing() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("summary"); + std::fs::write( + &path, + "hello-test-demo-smoke PASS\n\ + hello-test-demo-command1 FAIL exit code 1\n\ + container SKIP required isolation not available\n\ + flaky-one FLAKY passed on retry\n", + ) + .unwrap(); + let tests = [test("smoke"), test("command1")]; + let outcomes = parse_summary(&path, &tests).unwrap(); + assert_eq!(outcomes.len(), 4); + assert_eq!(outcomes[0].name, "smoke"); + assert_eq!(outcomes[0].verdict, Verdict::Pass); + assert_eq!(outcomes[1].verdict, Verdict::Fail); + assert_eq!(outcomes[1].reason.as_deref(), Some("exit code 1")); + assert_eq!(outcomes[2].name, "container"); + assert_eq!(outcomes[2].verdict, Verdict::Skip); + assert_eq!(outcomes[3].verdict, Verdict::Flaky); + } + + /// A missing summary file parses to no outcomes: the raw exit code + /// carries the verdict. + #[test] + fn missing_summary_yields_no_outcomes() { + let dir = tempfile::tempdir().unwrap(); + let outcomes = parse_summary(&dir.path().join("absent"), &[]).unwrap(); + assert!(outcomes.is_empty()); + } +}