diff --git a/src/test/mod.rs b/src/test/mod.rs index 1fade65..359bbca 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -712,4 +712,135 @@ mod tests { assert!(dir.is_dir()); fs::remove_dir(&dir).ok(); } + + /// The differential harness: pkh's native verdicts must match the + /// installed autopkgtest's on the same tree and binaries. This is + /// the trust anchor for the native runner — it pins the semantics + /// the DEP-8 documents under-specify. Deliberate, heavy and + /// host-mutating (both runners fall back to the host testbed), so + /// it only runs with --ignored and a tree to work on: + /// + /// ```sh + /// PKH_TEST_DIFF_TREE=../hello cargo test -- --ignored \ + /// test::tests::differential_harness_against_autopkgtest + /// ``` + #[test] + fn differential_harness_against_autopkgtest() { + let Some(tree) = std::env::var("PKH_TEST_DIFF_TREE") + .ok() + .map(PathBuf::from) + .map(|tree| tree.canonicalize().unwrap()) + else { + return; // not requested: nothing to compare + }; + + // pkh's native runner, on the host context (autopkgtest's null + // runner equivalent). + let rt = tokio::runtime::Runtime::new().unwrap(); + let native = rt.block_on(crate::test::run(TestOptions { + tree: Some(tree.clone()), + mode: Some(TestMode::Local), + ..Default::default() + })); + let native = match native { + Ok(report) => report, + Err(e) => panic!("native run failed: {e}"), + }; + let native_map: HashMap = native + .outcomes + .iter() + .map(|outcome| (outcome.name.clone(), outcome.verdict)) + .collect(); + + // The installed autopkgtest, same tree, same binaries, null + // runner (no isolation, like pkh's local mode). + let dir = tempfile::tempdir().unwrap(); + let summary = dir.path().join("summary"); + let mut command = std::process::Command::new("autopkgtest"); + command + .arg("-o") + .arg(dir.path()) + .arg("--summary") + .arg(&summary); + for deb in &native.binaries { + command.arg(deb); + } + command.arg(&tree).arg("--").arg("null"); + let status = command.status().expect("autopkgtest must be installed"); + let raw_code = status.code().unwrap_or(16); + assert!( + !matches!(raw_code, 12 | 14 | 16 | 20), + "autopkgtest failed outright (exit {raw_code})" + ); + + let tests_control = TestsControl::parse(&tree.join("debian/tests/control")).unwrap(); + let wrapped = vm::parse_summary(&summary, &tests_control.tests).unwrap(); + let wrapped_map: HashMap = wrapped + .iter() + .map(|outcome| (outcome.name.clone(), outcome.verdict)) + .collect(); + + let delta = verdict_delta(&native_map, &wrapped_map); + assert!( + delta.is_empty(), + "native and autopkgtest disagree:\n{}", + delta.join("\n") + ); + } + + /// Compare per-test verdict maps: failures first (a verdict class + /// change matters), then tests only one runner reported. Returns one + /// human-readable line per difference. + fn verdict_delta( + native: &HashMap, + wrapped: &HashMap, + ) -> Vec { + let mut delta = Vec::new(); + for (name, verdict) in native { + match wrapped.get(name) { + Some(wrapped) if wrapped == verdict => {} + Some(wrapped) => delta.push(format!( + "{name}: pkh={}, autopkgtest={}", + verdict.as_str(), + wrapped.as_str() + )), + None => delta.push(format!( + "{name}: pkh={}, autopkgtest did not report it", + verdict.as_str() + )), + } + } + for (name, verdict) in wrapped { + if !native.contains_key(name) { + delta.push(format!( + "{name}: autopkgtest={}, pkh did not report it", + verdict.as_str() + )); + } + } + delta.sort(); + delta + } + + #[test] + fn verdict_delta_reports_only_disagreements() { + let native = HashMap::from([ + ("passing".to_string(), Verdict::Pass), + ("failing".to_string(), Verdict::Fail), + ("pkh-only".to_string(), Verdict::Skip), + ]); + let wrapped = HashMap::from([ + ("passing".to_string(), Verdict::Pass), + ("failing".to_string(), Verdict::Skip), + ("wrapped-only".to_string(), Verdict::Fail), + ]); + let delta = verdict_delta(&native, &wrapped); + assert_eq!(delta.len(), 3, "{delta:?}"); + assert!( + delta[0].contains("failing: pkh=fail, autopkgtest=skip"), + "{delta:?}" + ); + assert!(delta[1].contains("pkh-only"), "{delta:?}"); + assert!(delta[2].contains("wrapped-only"), "{delta:?}"); + } }