From d35ffefacbaf4b32272c9c32b829b8182b8d6878 Mon Sep 17 00:00:00 2001
From: Valentin Haudiquet <valentin.haudiquet@canonical.com>
Date: Sun, 27 Sep 2026 21:16:31 +0200
Subject: [PATCH] test: add the differential harness against autopkgtest
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The native runner's trust anchor: on a tree of the caller's choosing
(PKH_TEST_DIFF_TREE, --ignored, deliberate: both runs land on the host
testbed), the harness runs pkh's local mode and the installed
autopkgtest with its null runner over the same tree and binaries, then
fails on any per-test verdict disagreement — pinning the semantics the
DEP-8 documents under-specify (verdict classes, argid names, edge-case
expansions). The comparison itself is pure and always tested.
---
 src/test/mod.rs | 131 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 131 insertions(+)

diff --git a/src/test/mod.rs b/src/test/mod.rs
index 1fade65..359bbca 100644
--- a/src/test/mod.rs
+++ b/src/test/mod.rs
@@ -712,4 +712,135 @@ mod tests {
         assert!(dir.is_dir());
         fs::remove_dir(&dir).ok();
     }
+
+    /// The differential harness: pkh's native verdicts must match the
+    /// installed autopkgtest's on the same tree and binaries. This is
+    /// the trust anchor for the native runner — it pins the semantics
+    /// the DEP-8 documents under-specify. Deliberate, heavy and
+    /// host-mutating (both runners fall back to the host testbed), so
+    /// it only runs with --ignored and a tree to work on:
+    ///
+    /// ```sh
+    /// PKH_TEST_DIFF_TREE=../hello cargo test -- --ignored \
+    ///     test::tests::differential_harness_against_autopkgtest
+    /// ```
+    #[test]
+    fn differential_harness_against_autopkgtest() {
+        let Some(tree) = std::env::var("PKH_TEST_DIFF_TREE")
+            .ok()
+            .map(PathBuf::from)
+            .map(|tree| tree.canonicalize().unwrap())
+        else {
+            return; // not requested: nothing to compare
+        };
+
+        // pkh's native runner, on the host context (autopkgtest's null
+        // runner equivalent).
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let native = rt.block_on(crate::test::run(TestOptions {
+            tree: Some(tree.clone()),
+            mode: Some(TestMode::Local),
+            ..Default::default()
+        }));
+        let native = match native {
+            Ok(report) => report,
+            Err(e) => panic!("native run failed: {e}"),
+        };
+        let native_map: HashMap<String, Verdict> = native
+            .outcomes
+            .iter()
+            .map(|outcome| (outcome.name.clone(), outcome.verdict))
+            .collect();
+
+        // The installed autopkgtest, same tree, same binaries, null
+        // runner (no isolation, like pkh's local mode).
+        let dir = tempfile::tempdir().unwrap();
+        let summary = dir.path().join("summary");
+        let mut command = std::process::Command::new("autopkgtest");
+        command
+            .arg("-o")
+            .arg(dir.path())
+            .arg("--summary")
+            .arg(&summary);
+        for deb in &native.binaries {
+            command.arg(deb);
+        }
+        command.arg(&tree).arg("--").arg("null");
+        let status = command.status().expect("autopkgtest must be installed");
+        let raw_code = status.code().unwrap_or(16);
+        assert!(
+            !matches!(raw_code, 12 | 14 | 16 | 20),
+            "autopkgtest failed outright (exit {raw_code})"
+        );
+
+        let tests_control = TestsControl::parse(&tree.join("debian/tests/control")).unwrap();
+        let wrapped = vm::parse_summary(&summary, &tests_control.tests).unwrap();
+        let wrapped_map: HashMap<String, Verdict> = wrapped
+            .iter()
+            .map(|outcome| (outcome.name.clone(), outcome.verdict))
+            .collect();
+
+        let delta = verdict_delta(&native_map, &wrapped_map);
+        assert!(
+            delta.is_empty(),
+            "native and autopkgtest disagree:\n{}",
+            delta.join("\n")
+        );
+    }
+
+    /// Compare per-test verdict maps: failures first (a verdict class
+    /// change matters), then tests only one runner reported. Returns one
+    /// human-readable line per difference.
+    fn verdict_delta(
+        native: &HashMap<String, Verdict>,
+        wrapped: &HashMap<String, Verdict>,
+    ) -> Vec<String> {
+        let mut delta = Vec::new();
+        for (name, verdict) in native {
+            match wrapped.get(name) {
+                Some(wrapped) if wrapped == verdict => {}
+                Some(wrapped) => delta.push(format!(
+                    "{name}: pkh={}, autopkgtest={}",
+                    verdict.as_str(),
+                    wrapped.as_str()
+                )),
+                None => delta.push(format!(
+                    "{name}: pkh={}, autopkgtest did not report it",
+                    verdict.as_str()
+                )),
+            }
+        }
+        for (name, verdict) in wrapped {
+            if !native.contains_key(name) {
+                delta.push(format!(
+                    "{name}: autopkgtest={}, pkh did not report it",
+                    verdict.as_str()
+                ));
+            }
+        }
+        delta.sort();
+        delta
+    }
+
+    #[test]
+    fn verdict_delta_reports_only_disagreements() {
+        let native = HashMap::from([
+            ("passing".to_string(), Verdict::Pass),
+            ("failing".to_string(), Verdict::Fail),
+            ("pkh-only".to_string(), Verdict::Skip),
+        ]);
+        let wrapped = HashMap::from([
+            ("passing".to_string(), Verdict::Pass),
+            ("failing".to_string(), Verdict::Skip),
+            ("wrapped-only".to_string(), Verdict::Fail),
+        ]);
+        let delta = verdict_delta(&native, &wrapped);
+        assert_eq!(delta.len(), 3, "{delta:?}");
+        assert!(
+            delta[0].contains("failing: pkh=fail, autopkgtest=skip"),
+            "{delta:?}"
+        );
+        assert!(delta[1].contains("pkh-only"), "{delta:?}");
+        assert!(delta[2].contains("wrapped-only"), "{delta:?}");
+    }
 }