//! Run a source package's DEP-8 as-installed tests (`pkh test`). //! //! The runner is native for the chroot and local testbeds: the testbed //! is a pkh context (an ephemeral unshare chroot bootstrapped exactly //! like a `pkh deb` build chroot, or the current context directly), and //! the DEP-8 per-test contract lives in [`runner`]. `--mode vm` execs //! the installed autopkgtest with its qemu runner instead — the one //! backend pkh does not reimplement (serial-console protocol, image //! management, reboot support). //! //! Binaries under test are the fresh `pkh deb` output next to the tree //! (freshness-guarded like the lint wrapper); when they are missing or //! stale, the deb flow builds them in-process first. pub(crate) mod bed; pub mod expand; pub mod options; pub mod outcome; pub(crate) mod runner; pub(crate) mod vm; pub use options::{TestMode, TestOptions}; pub use outcome::TestRunReport; use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; use std::sync::Arc; use crate::context::{LineSink, Stream}; use crate::debian::control::{ControlInfo, parse_paragraphs}; use crate::debian::tests_control::TestsControl; use crate::logfmt::GenericClassifier; use crate::report::{BuildTarget, BuildView, TargetKind}; use crate::test::bed::Bed; use crate::test::outcome::Verdict; /// Where the built binaries under test came from. struct Binaries { /// Host-side paths of the .deb files. paths: Vec, /// Provenance note for the report. note: String, } /// Run the DEP-8 tests of a source package tree. /// /// Errors are runtime errors (no declared tests, unusable testbed, /// failed in-process build); test failures are a report, not an error. pub async fn run(opts: TestOptions<'_>) -> Result> { let view = opts.view; let tree = opts .tree .clone() .unwrap_or_else(|| std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."))); let tree = tree .canonicalize() .map_err(|e| format!("cannot access the source tree '{}': {e}", tree.display()))?; // Package identity and target, resolved like a deb build. let entry = crate::debian::parse_changelog_entry(&tree.join("debian/changelog")) .map_err(|e| format!("cannot read the package identity: {e}"))?; let package = entry.source.clone(); let version = entry.version.full(); let dist = opts .dist .clone() .unwrap_or_else(crate::build::env::current_vendor); let series = match opts.series.clone() { Some(series) => series, None if crate::distro_info::is_unreleased(&entry.distribution) => { let resolved = crate::distro_info::effective_series(&entry.distribution, &dist).await?; log::info!( "Changelog is UNRELEASED, testing against series {}", resolved ); resolved } None => entry.distribution.clone(), }; let arch = opts.arch.clone().unwrap_or_else(crate::get_current_arch); // Pre-check: the package must declare tests, before any testbed is // booted. let control_path = tree.join("debian/tests/control"); if !control_path.exists() { return Err( "the package declares no autopkgtests (no debian/tests/control). \ Add debian/tests/control and 'Testsuite: autopkgtest' to debian/control \ (pkh lint flags this too), or point pkh test at another tree." .into(), ); } let tests_control = TestsControl::parse(&control_path) .map_err(|e| format!("cannot parse debian/tests/control: {e}"))?; if tests_control.tests.is_empty() { return Err("debian/tests/control declares no tests".into()); } // Binaries under test: fresh pkh deb output, an in-process build, or // an explicit --debs list. let binaries = resolve_binaries(&opts, &tree, &package, &version, &arch, view).await?; view.target(BuildTarget { package: &package, version: &version, target: &format!("{series}/{arch}"), display: format!( "Testing {} ({}) on {}/{} ({})", package, version, series, arch, opts.mode.unwrap_or_default().label() ), kind: TargetKind::Test, tee_log: true, }); let mode = opts.mode.unwrap_or_default(); // The vm escape hatch goes through the installed autopkgtest; it // owns the artifacts directory (passed as -o, which must not exist // yet), so the path is only computed, not created. if mode == TestMode::Vm { let run_dir = run_dir_path(&package, &series, &arch); let (outcomes, raw_code) = vm::run_vm( &tree, &binaries.paths, &tests_control.tests, &run_dir, opts.image.as_deref(), &opts, view, )?; return finish( TestRunReport { package, version, series, arch, mode: mode.label().to_string(), binaries: binaries.paths, binaries_note: binaries.note, outcomes, artifacts_dir: run_dir, raw_exit_code: Some(raw_code), }, view, ); } if mode == TestMode::Local { log::warn!( "Tests run directly on the current context: they install packages \ and can rewrite it. Use the default chroot mode for isolation." ); } let run_dir = create_run_dir(&package, &series, &arch)?; let sink = view.sink().or_else(|| { // Verbose runs without a live view still stream test output live. opts.verbose .then(|| Arc::new(TerminalLines) as Arc) }); let base_ctx = crate::context::current(); let bed = Bed::prepare(mode, &series, &arch, base_ctx, view).await?; let ctx = bed.ctx(); let mut env = HashMap::new(); env.insert("LANG".to_string(), "C".to_string()); env.insert("DEBIAN_FRONTEND".to_string(), "noninteractive".to_string()); if matches!(bed, Bed::Chroot(_)) { env.insert("HOME".to_string(), "/root".to_string()); env.insert("USER".to_string(), "root".to_string()); env.insert("LOGNAME".to_string(), "root".to_string()); } configure_testbed( &ctx, &env, &series, &arch, &opts, matches!(bed, Bed::Local(_)), view, ) .await?; // Stage the tree and the binaries into the testbed. The copy also // protects the host tree: tests may write into it (rw-build-tree). view.message("Staging the tree and binaries"); let staging_root = bed.scratch_root()?; let staged_tree = ctx.ensure_available(&tree, &staging_root)?; let staged_tree = staged_tree.to_string_lossy().into_owned(); let mut staged_debs = Vec::new(); for deb in &binaries.paths { staged_debs.push(ctx.ensure_available(deb, &staging_root)?); } // Install the exact binaries under test before the tests run. if !staged_debs.is_empty() { view.phase( "Installing built binaries", Box::new(GenericClassifier::new()), ); let mut cmd = ctx.command("apt-get"); cmd.envs(env.clone()) .arg("-y") .arg("--allow-downgrades") .arg("install"); for deb in &staged_debs { cmd.arg(format!( "./{}", deb.to_string_lossy().trim_start_matches('/') )); } let status = cmd.status().map_err(|e| e.to_string())?; if !status.success() { return Err("installing the built binaries into the testbed failed".into()); } } // What @ expands to: the names of the binaries actually under test, // from their .deb file names (__.deb). let control_info = ControlInfo::parse(&tree.join("debian/control")) .map_err(|e| format!("cannot parse debian/control: {e}"))?; let binary_names: Vec = staged_debs .iter() .filter_map(|deb| deb_binary_name(deb)) .collect(); let run_ctx = runner::RunContext { ctx, tree: staged_tree.clone(), staging_root, arch: arch.clone(), host_arch: crate::get_current_arch(), supports_root: bed.supports_root(), timeout_secs: opts.timeout_secs(), sink, view, control: control_info, binaries: binary_names, env: env.clone(), artifacts_dir: run_dir.clone(), }; let outcomes = runner::run_tests( &run_ctx, &tests_control.tests, &opts.test_names, &opts.skip_tests, ); // Debugging escape hatches. let has_failures = outcomes .iter() .any(|outcome| outcome.verdict == Verdict::Fail); if opts.shell || (opts.shell_fail && has_failures) { drop_shell(&bed, &staged_tree, &env, view)?; } finish( TestRunReport { package, version, series, arch, mode: mode.label().to_string(), binaries: binaries.paths, binaries_note: binaries.note, outcomes, artifacts_dir: run_dir, raw_exit_code: None, }, view, ) } /// Announce the outcome on the view and hand the report back. fn finish( report: TestRunReport, view: &dyn BuildView, ) -> Result> { if report.has_failures() { view.finish_failure(); } else { view.finish_success(std::slice::from_ref(&report.artifacts_dir)); } Ok(report) } /// Resolve the binaries under test: `--debs` overrides everything; /// otherwise the fresh `pkh deb` output next to the tree (freshness- /// guarded, the lint wrapper's artifact strategy); otherwise the deb /// flow builds them in-process. `--no-build` refuses instead. async fn resolve_binaries( opts: &TestOptions<'_>, tree: &Path, package: &str, version: &str, arch: &str, view: &dyn BuildView, ) -> Result> { if !opts.debs.is_empty() { for deb in &opts.debs { if !deb.exists() { return Err(format!("--deb file '{}' does not exist", deb.display()).into()); } } return Ok(Binaries { paths: opts.debs.clone(), note: "explicitly passed with --debs".to_string(), }); } if !opts.build && let Some(paths) = fresh_deb_output(tree, package, version, arch) { log::info!("Testing the fresh pkh deb output next to the tree"); return Ok(Binaries { paths, note: "fresh pkh deb output".to_string(), }); } if opts.no_build { return Err( "no fresh pkh deb output next to the tree and --no-build given: \ run 'pkh deb' first, drop --no-build, or pass --debs" .into(), ); } view.message("No fresh pkh deb output: building the binaries first"); let artifacts = crate::deb::build_binary_package(crate::deb::DebBuildOptions { series: opts.series.clone(), arch: opts.arch.clone(), pocket: opts.pocket.clone(), ppa: opts.ppa.clone(), cwd: Some(tree.to_path_buf()), view, ..Default::default() }) .await?; let debs: Vec = artifacts .into_iter() .filter(|path| path.extension().is_some_and(|ext| ext == "deb")) .collect(); if debs.is_empty() { return Err("the build produced no binary packages to test".into()); } Ok(Binaries { paths: debs, note: "built by pkh deb just now".to_string(), }) } /// The fresh `pkh deb` output next to the tree, when it matches the /// current changelog entry and no tree content is newer than it: the /// `__.changes` file gates, its checksum lists /// name the debs. Stale or missing output tests nothing. fn fresh_deb_output(tree: &Path, package: &str, version: &str, arch: &str) -> Option> { let entry = crate::debian::parse_changelog_entry(&tree.join("debian/changelog")).ok()?; let parent = tree.parent()?; let changes_path = parent.join(format!( "{}_{}_{}.changes", package, entry.version.no_epoch(), arch )); let changes = fs::read_to_string(&changes_path).ok()?; // Only the current entry's binaries are the ones under test. let paragraph = parse_paragraphs(&changes).into_iter().next()?; let matches = paragraph.get("Source").map(str::trim) == Some(package) && paragraph.get("Version").map(str::trim) == Some(version); if !matches { return None; } let mut debs = Vec::new(); for (field, value) in paragraph.iter() { if !field.starts_with("Checksums-") && field != "Files" { continue; } for line in value.lines() { // `
[] ` if let Some(name) = line.split_whitespace().last() && name.ends_with(".deb") { debs.push(parent.join(name)); } } } if debs.is_empty() || debs.iter().any(|deb| !deb.exists()) { return None; } // A tree file written after the changes file makes the output stale. let built = fs::metadata(&changes_path).ok()?.modified().ok()?; if tree_newer_than(tree, built) { return None; } Some(debs) } /// Whether any tree content is newer than `built`. Skips `.git` and /// `.pc`: commits and quilt bookkeeping churn their mtimes without /// touching what the package contains. fn tree_newer_than(root: &Path, built: std::time::SystemTime) -> bool { let mut stack = vec![root.to_path_buf()]; while let Some(dir) = stack.pop() { let Ok(entries) = fs::read_dir(&dir) else { continue; }; for entry in entries.flatten() { let path = entry.path(); let name = entry.file_name(); if name == ".git" || name == ".pc" { continue; } let Ok(meta) = fs::symlink_metadata(&path) else { continue; }; if meta.is_dir() { stack.push(path); } else if meta.modified().is_ok_and(|mtime| mtime > built) { return true; } } } false } /// Configure the testbed's APT setup and update it: pockets and PPAs on /// the archive sources (the same rules as the deb build), an optional /// upgrade, the caller's setup commands and injected packages. async fn configure_testbed( ctx: &Arc, env: &HashMap, series: &str, arch: &str, opts: &TestOptions<'_>, local_context: bool, view: &dyn BuildView, ) -> Result<(), Box> { let mut sources = crate::apt::sources::load(Some(ctx.clone()))?; let mut modified = false; let mut added_ppas: Vec<(&str, &str)> = Vec::new(); for ppa in &opts.ppa { let (ppa_user, ppa_name) = crate::package_info::split_ppa(ppa)?; let base_url = crate::package_info::ppa_to_base_url(ppa_user, ppa_name); if sources.iter().any(|s| s.uri.contains(&base_url)) { continue; } let mut architectures = vec![crate::get_current_arch()]; if arch != crate::get_current_arch() { architectures.push(arch.to_string()); } sources.push(crate::apt::sources::SourceEntry { enabled: true, kind: crate::apt::sources::SourceKind::Deb, components: vec!["main".to_string()], architectures, signed_by: None, trusted: None, suite: vec![series.to_string()], uri: base_url, origin: None, }); added_ppas.push((ppa_user, ppa_name)); modified = true; } // Ubuntu: test dependencies live in universe as often as in main. let ubuntu_components = crate::distro_info::get_dist_components("ubuntu")?; if ubuntu_components.iter().any(|c| c == "universe") { for source in &mut sources { if crate::distro_info::is_official_source("ubuntu", &source.uri) && !source.components.contains(&"universe".to_string()) { source.components.push("universe".to_string()); modified = true; } } } if let Some(pocket) = &opts.pocket { let pocket_suite = format!("{series}-{pocket}"); log::info!("Enabling pocket '{}' for test dependencies", pocket_suite); for source in &mut sources { if crate::deb::is_archive_source(&source.uri) && !source.suite.contains(&pocket_suite) { source.suite.push(pocket_suite.clone()); modified = true; } } // proposed is NotAutomatic: without a pin apt would ignore it. if pocket.starts_with("proposed") { crate::deb::local::pin_pocket(&pocket_suite, ctx)?; } } if modified { crate::apt::sources::save(Some(ctx.clone()), sources)?; for (user, ppa_name) in added_ppas { if let Err(e) = crate::apt::keyring::download_trust_ppa_key(Some(ctx.clone()), user, ppa_name).await { log::warn!("Failed to download PPA key for {user}/{ppa_name}: {e}"); } } } view.phase("Configuring testbed", Box::new(GenericClassifier::new())); let status = ctx .command("apt-get") .envs(env.clone()) .arg("update") .status() .map_err(|e| format!("apt-get update failed inside the testbed: {e}"))?; if !status.success() { return Err(format!( "apt-get update failed inside the testbed{}", if local_context { " (a local run needs root to install packages: \ use sudo, or the default chroot mode)" } else { "" } ) .into()); } if opts.apt_upgrade { for action in ["upgrade", "autopurge"] { let status = ctx .command("apt-get") .envs(env.clone()) .arg("-y") .arg(action) .status() .map_err(|e| e.to_string())?; if !status.success() { return Err(format!("apt-get {action} failed inside the testbed").into()); } } } for setup in &opts.setup_commands { let status = ctx .command("bash") .envs(env.clone()) .arg("-e") .arg("-c") .arg(setup) .status() .map_err(|e| e.to_string())?; if !status.success() { return Err(format!("setup command failed: {setup}").into()); } } if !opts.inject.is_empty() { // Testbed setup is not a profiled build: the durations and the // phase profile go nowhere. crate::deb::local::install_injected_packages( &opts.inject, env, ctx.clone(), view, &None, &mut crate::deb::session::BuildTimings::default(), &mut crate::deb::BuildProfiler::new(), )?; } Ok(()) } /// Drop into an interactive testbed shell (the `--shell`/ /// `--shell-fail` debugging escape hatch). fn drop_shell( bed: &Bed, staged_tree: &str, env: &HashMap, view: &dyn BuildView, ) -> Result<(), Box> { view.suspend(); log::info!("Dropping into the testbed shell (exit to continue)"); let status = bed .ctx() .command("bash") .envs(env.iter()) .current_dir(staged_tree) .status()?; if !status.success() { log::warn!("Testbed shell exited with {}", status); } Ok(()) } /// Create the run's host artifacts directory: /// `/var/tmp/pkh/tests/---/`. fn create_run_dir(package: &str, series: &str, arch: &str) -> std::io::Result { let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%S"); let dir = runs_root().join(format!("{package}-{series}-{arch}-{timestamp}")); fs::create_dir_all(&dir)?; Ok(dir) } /// The run directory path for an identity, without creating it. fn run_dir_path(package: &str, series: &str, arch: &str) -> PathBuf { let timestamp = chrono::Utc::now().format("%Y%m%dT%H%M%S"); runs_root().join(format!("{package}-{series}-{arch}-{timestamp}")) } /// The test-run artifacts root: `$PKH_TESTS_DIR`, defaulting to /// `/var/tmp/pkh/tests`. pub fn runs_root() -> PathBuf { if let Ok(dir) = std::env::var("PKH_TESTS_DIR") && !dir.is_empty() { return PathBuf::from(dir); } PathBuf::from("/var/tmp/pkh/tests") } /// Render the tests declared in the tree's `debian/tests/control`, one /// block per test (`pkh test --list-tests`). pub fn list_tests(tree: &Path) -> Result> { let tests_control = TestsControl::parse(&tree.join("debian/tests/control")) .map_err(|e| format!("cannot parse debian/tests/control: {e}"))?; let mut out = String::new(); for test in &tests_control.tests { out.push_str(&test.name); out.push('\n'); if let Some(command) = &test.command { out.push_str(&format!(" Test-Command: {command}\n")); } if !test.restrictions.is_empty() { out.push_str(&format!( " Restrictions: {}\n", test.restrictions.join(" ") )); } if let Some(depends) = &test.depends { out.push_str(&format!(" Depends: {depends}\n")); } if let Some(directory) = &test.tests_directory { out.push_str(&format!(" Tests-Directory: {directory}\n")); } } Ok(out) } /// The binary package name of a staged .deb file name. fn deb_binary_name(path: &Path) -> Option { let name = path.file_name()?.to_string_lossy(); let stem = name.strip_suffix(".deb")?; Some(stem.split('_').next()?.to_string()) } /// Live terminal passthrough of captured lines, for `--verbose` runs /// without a live view. struct TerminalLines; impl LineSink for TerminalLines { fn line(&self, stream: Stream, line: &str) { match stream { Stream::Stdout => println!("{line}"), Stream::Stderr => eprintln!("{line}"), } } } #[cfg(test)] mod tests { use super::*; /// The freshness guard accepts the changes file matching the current /// changelog entry with its debs, and rejects a stale one. #[test] fn fresh_deb_output_freshness() { let base = tempfile::tempdir().unwrap(); let tree = base.path().join("hello"); fs::create_dir_all(tree.join("debian")).unwrap(); fs::write( tree.join("debian/changelog"), "hello (2.10-4) noble; urgency=medium\n\n * bump\n\n -- A Thu, 01 Jan 2026 00:00:00 +0000\n", ) .unwrap(); let changes_name = "hello_2.10-4_amd64.changes"; let write_changes = |fresh: bool| { let content = format!( "Format: 1.8\nSource: hello\nVersion: 2.10-4\nChecksums-Sha256:\n \ abc 123 utils {} \n", changes_name.replace(".changes", ".deb") ); let path = base.path().join(changes_name); fs::write(&path, content).unwrap(); if !fresh { // Push the tree strictly newer than the changes file. let newer = tree.join("debian/control"); fs::write(&newer, "Source: hello\n").unwrap(); let time = std::time::SystemTime::now() + std::time::Duration::from_secs(3600); let file = fs::File::options().append(true).open(&newer).unwrap(); file.set_times(std::fs::FileTimes::new().set_modified(time)) .unwrap(); } }; // No changes file at all: nothing to reuse. assert!(fresh_deb_output(&tree, "hello", "2.10-4", "amd64").is_none()); // The deb the changes file lists must exist for the reuse. fs::write(base.path().join("hello_2.10-4_amd64.deb"), "deb").unwrap(); write_changes(true); let debs = fresh_deb_output(&tree, "hello", "2.10-4", "amd64").unwrap(); assert_eq!(debs, vec![base.path().join("hello_2.10-4_amd64.deb")]); // A different version or arch tests nothing. assert!(fresh_deb_output(&tree, "hello", "2.10-3", "amd64").is_none()); assert!(fresh_deb_output(&tree, "hello", "2.10-4", "arm64").is_none()); // Tree content newer than the output makes it stale. write_changes(false); assert!(fresh_deb_output(&tree, "hello", "2.10-4", "amd64").is_none()); } /// The run directory is created fresh under the pkh tests root. #[test] fn run_dir_layout() { let dir = create_run_dir("hello", "noble", "amd64").unwrap(); assert_eq!(dir.parent(), Some(Path::new("/var/tmp/pkh/tests"))); assert!( dir.file_name() .and_then(|name| name.to_str()) .is_some_and(|name| name.starts_with("hello-noble-amd64-")) ); assert!(dir.is_dir()); fs::remove_dir(&dir).ok(); } /// The differential harness: pkh's native verdicts must match the /// installed autopkgtest's on the same tree and binaries. This is /// the trust anchor for the native runner — it pins the semantics /// the DEP-8 documents under-specify. Deliberate, heavy and /// host-mutating (both runners fall back to the host testbed), so /// it only runs with --ignored and a tree to work on: /// /// ```sh /// PKH_TEST_DIFF_TREE=../hello cargo test -- --ignored \ /// test::tests::differential_harness_against_autopkgtest /// ``` #[test] fn differential_harness_against_autopkgtest() { let Some(tree) = std::env::var("PKH_TEST_DIFF_TREE") .ok() .map(PathBuf::from) .map(|tree| tree.canonicalize().unwrap()) else { return; // not requested: nothing to compare }; // pkh's native runner, on the host context (autopkgtest's null // runner equivalent). let rt = tokio::runtime::Runtime::new().unwrap(); let native = rt.block_on(crate::test::run(TestOptions { tree: Some(tree.clone()), mode: Some(TestMode::Local), ..Default::default() })); let native = match native { Ok(report) => report, Err(e) => panic!("native run failed: {e}"), }; let native_map: HashMap = native .outcomes .iter() .map(|outcome| (outcome.name.clone(), outcome.verdict)) .collect(); // The installed autopkgtest, same tree, same binaries, null // runner (no isolation, like pkh's local mode). let dir = tempfile::tempdir().unwrap(); let summary = dir.path().join("summary"); let mut command = std::process::Command::new("autopkgtest"); command .arg("-o") .arg(dir.path()) .arg("--summary") .arg(&summary); for deb in &native.binaries { command.arg(deb); } command.arg(&tree).arg("--").arg("null"); let status = command.status().expect("autopkgtest must be installed"); let raw_code = status.code().unwrap_or(16); assert!( !matches!(raw_code, 12 | 14 | 16 | 20), "autopkgtest failed outright (exit {raw_code})" ); let tests_control = TestsControl::parse(&tree.join("debian/tests/control")).unwrap(); let wrapped = vm::parse_summary(&summary, &tests_control.tests).unwrap(); let wrapped_map: HashMap = wrapped .iter() .map(|outcome| (outcome.name.clone(), outcome.verdict)) .collect(); let delta = verdict_delta(&native_map, &wrapped_map); assert!( delta.is_empty(), "native and autopkgtest disagree:\n{}", delta.join("\n") ); } /// Compare per-test verdict maps: failures first (a verdict class /// change matters), then tests only one runner reported. Returns one /// human-readable line per difference. fn verdict_delta( native: &HashMap, wrapped: &HashMap, ) -> Vec { let mut delta = Vec::new(); for (name, verdict) in native { match wrapped.get(name) { Some(wrapped) if wrapped == verdict => {} Some(wrapped) => delta.push(format!( "{name}: pkh={}, autopkgtest={}", verdict.as_str(), wrapped.as_str() )), None => delta.push(format!( "{name}: pkh={}, autopkgtest did not report it", verdict.as_str() )), } } for (name, verdict) in wrapped { if !native.contains_key(name) { delta.push(format!( "{name}: autopkgtest={}, pkh did not report it", verdict.as_str() )); } } delta.sort(); delta } #[test] fn verdict_delta_reports_only_disagreements() { let native = HashMap::from([ ("passing".to_string(), Verdict::Pass), ("failing".to_string(), Verdict::Fail), ("pkh-only".to_string(), Verdict::Skip), ]); let wrapped = HashMap::from([ ("passing".to_string(), Verdict::Pass), ("failing".to_string(), Verdict::Skip), ("wrapped-only".to_string(), Verdict::Fail), ]); let delta = verdict_delta(&native, &wrapped); assert_eq!(delta.len(), 3, "{delta:?}"); assert!( delta[0].contains("failing: pkh=fail, autopkgtest=skip"), "{delta:?}" ); assert!(delta[1].contains("pkh-only"), "{delta:?}"); assert!(delta[2].contains("wrapped-only"), "{delta:?}"); } }