diff --git a/tests/e2e/.gitignore b/tests/e2e/.gitignore new file mode 100644 index 0000000..aca4e0f --- /dev/null +++ b/tests/e2e/.gitignore @@ -0,0 +1,2 @@ +# Harness run output: trees, logs, results +runs/ diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 0000000..110872c --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,103 @@ +# pkh end-to-end package harness + +Exercises pkh the way it is used in anger: real packages, the whole +workflow. For every package in `catalog.yml` it runs + +``` +pkh pull --archive # sources from the Debian/Ubuntu archive +pkh chlog --rebuild # no-change rebuild entry (1.0-1 -> 1.0-1build1) +pkh build # .dsc + _source.changes next to the tree +pkh deb # binary packages in an ephemeral chroot +``` + +The catalog spans ~60 source packages: smoke-test smalls (`hello`, +`micro`, `jq`...), core utilities, server software, databases, desktop +stacks, toolchains and the weird corners — `dracut`, `opensbi`, `grub2`, +a cross `u-boot`, the kbuild-based `kbuild`, and (opt-in, hours each) +`gcc-14`, `glibc`, `rustc`, `libreoffice`, `firefox`, the Ubuntu `linux` +kernel... + +## Prerequisites + +- A pkh binary: the harness prefers the repo build + (`target/release/pkh`, then `target/debug/pkh`), falls back to `pkh` + on PATH; override with `--pkh PATH` or build first with `--build-pkh`. +- The pkh runtime toolchain: `mmdebstrap`, `sudo` (passwordless, for + the unshare chroot builds), `dpkg-dev`, `distro-info` data, and + network access to the Debian and Ubuntu archives. +- Python 3 only (stdlib; PyYAML is used when installed, with a built-in + fallback parser for the catalog's YAML subset). +- Disk: tens of GB under `runs/` for trees and artifacts, plus pkh's own + caches (`/var/tmp/pkh`); expect a full default-tier run to take many + hours — the tiers exist so you never have to. + +## Usage + +``` +./run.py --list # what would run, with tiers and notes +./run.py --dry-run # the exact commands, nothing executed +./run.py # smoke + standard tiers +./run.py --tier long # the multi-hour builds +./run.py --tier all # everything +./run.py --only hello,dracut # exact names +./run.py --category database # or server, desktop, boot-firmware... +./run.py --tag cross --arch arm64 # tag/arch sweeps +./run.py --stages pull # catalog sanity: can we pull everything? +./run.py --resume runs//results.json # continue an interrupted run +``` + +Useful global pass-throughs: `--dist`, `--series`, `--arch`, `--cross`, +`--pocket`, `--ppa user/ppa` (deb dependencies), `--jobs N`, +`--no-ccache`, `--git` (pull through the VCS instead of the archive). + +Exit codes: `0` nothing unexpected failed, `1` at least one package +failed (or xpassed under `--strict`), `2` usage/catalog error, `130` +interrupted. + +## Selection: tiers, categories, tags + +Entries carry a **tier**: `smoke` (minutes), `standard` (the default, +up to ~1-2h each), `long` (hours: compilers, browsers, the kernel — +only run when asked). Default selection is smoke + standard. On top of +that, filters compose: `--category`, `--tag`, `--only`, `--skip`. + +An entry can be marked `xfail: ` when it is known-broken; it +then reports `xfail` on failure and `xpass` on unexpected success +(xpass fails the run only under `--strict`). + +## What a run produces + +``` +runs// + logs/.log # full output of every stage of that package + // # the pulled source tree + /*.dsc ... # build artifacts, next to the tree as pkh puts them + results.json # machine-readable, written progressively + results.md # the summary table +``` + +The run directory is per invocation; `--resume /results.json` +reuses it, keeps the recorded outcomes and only reruns packages that +did not finish. Already-pulled trees are kept (`pkh pull` refuses +existing directories anyway — delete `runs///` to force a +re-pull), and stages that already succeeded are reused rather than +re-run, so a resumed `chlog` does not bump the version twice; change +`--dist`/`--series`/`--arch`/`--cross` and everything re-runs. + +Pulls run a few at a time (`--pull-jobs`), the build pipeline is +sequential by design: one heavy build at a time keeps pkh's shared +chroot tarball, apt and ccache caches warm, so repeat runs get +dramatically faster. + +Each stage gets the package's timeout budget (tier default, overridable +per entry with `timeout:`); on timeout the whole build process group is +killed and the package is marked `timeout`. + +## Adding packages + +Edit `catalog.yml`. Fields are documented at the top of the file; +keep **source** names as they exist in the archive — not binary names +(`make-dfsg` and `rust-ripgrep`, not `make` and `ripgrep`; +`xz-utils`, not `xz`; `postgresql-17`, not `postgresql`). The file +stays within a small YAML subset so the stdlib fallback parser can read +it: flat mappings, inline lists, no colons in values. diff --git a/tests/e2e/__pycache__/run.cpython-314.pyc b/tests/e2e/__pycache__/run.cpython-314.pyc new file mode 100644 index 0000000..ef52a8f Binary files /dev/null and b/tests/e2e/__pycache__/run.cpython-314.pyc differ diff --git a/tests/e2e/catalog.yml b/tests/e2e/catalog.yml new file mode 100644 index 0000000..3881d67 --- /dev/null +++ b/tests/e2e/catalog.yml @@ -0,0 +1,375 @@ +# pkh end-to-end test catalog +# +# One entry per source package, exercised by run.py as: +# pkh pull -> pkh chlog --rebuild -> pkh build -> pkh deb +# +# The catalog deliberately alternates between debian and ubuntu within +# every category, so a default run exercises both archives, series +# namings and chroots. +# +# The file is plain YAML but is also parsed by a small built-in parser when +# PyYAML is unavailable: keep to the subset it supports — full-line comments, +# 'key: value' scalars, inline [a, b] lists, no nested mappings, no block +# scalars, no colons inside values. +# +# Fields (all optional except 'name'): +# category free grouping label (smoke, server, desktop, ...) [defaults.tier] +# tier smoke | standard | long — 'long' is only run when asked for +# dist debian | ubuntu [defaults.dist] +# series pin a series (default: the dist development series) +# arch pass '-a ARCH' to 'pkh deb' +# cross true -> also pass '--cross' (needs 'arch') [false] +# pocket pass '-p POCKET' to pull/deb +# ppa inline list of PPAs passed to 'pkh deb --ppa' +# timeout per-stage budget in minutes [tier default] +# xfail known-broken marker: the reason it is expected to fail +# notes free text, shown by --list and in reports + +defaults: + dist: debian + tier: standard + +packages: + # --- smoke: small, fast, reliable; the first line of defense ------------- + - name: hello + category: smoke + tier: smoke + tags: [gnu, autotools, tiny] + notes: the canonical packaging smoke test + + - name: micro + category: smoke + tier: smoke + tags: [go, editor, tiny] + notes: Go build, golang build-dependencies resolved offline + + - name: jq + category: smoke + tier: smoke + tags: [c, autotools, tiny] + + - name: htop + category: smoke + dist: ubuntu + tier: smoke + tags: [c, autotools, tiny] + + - name: tmux + category: smoke + tier: smoke + tags: [c, autotools, tiny] + + - name: tree + category: smoke + dist: ubuntu + tier: smoke + tags: [c, make, tiny] + + - name: figlet + category: smoke + dist: ubuntu + tier: smoke + tags: [c, make, tiny] + + - name: dash + category: smoke + tier: smoke + tags: [c, autotools, shell] + + - name: file + category: smoke + tier: smoke + tags: [c, autotools] + + - name: zlib + category: smoke + dist: ubuntu + tier: smoke + tags: [c, autotools, lib] + + - name: xz-utils + category: smoke + tier: smoke + tags: [c, autotools] + xfail: sid carries the 5.8.1-1+deb13u1 trixie upload whose packaging fails at dh_installdocs (doxygen api docs missing) - package-side, under triage + + - name: busybox + category: smoke + dist: ubuntu + tier: smoke + tags: [c, kconfig, static] + + - name: dnsmasq + category: smoke + tier: smoke + tags: [c, make, network] + + - name: memcached + category: smoke + dist: ubuntu + tier: smoke + tags: [c, autotools, server] + + - name: kbuild + category: smoke + tier: smoke + tags: [c, kbuild, weird] + notes: the virtualbox build system; exercises the kbuild log classifier + + # --- core ----------------------------------------------------------------- + - name: bash + category: core + tags: [c, autotools, shell] + + - name: coreutils + category: core + dist: ubuntu + tags: [c, autotools] + xfail: pkh deb builds as root in the chroot and coreutils configure refuses root (needs a non-root build user, as Launchpad buildds use) + + - name: util-linux + category: core + tags: [c, autotools] + + - name: openssl + category: core + dist: ubuntu + tags: [c, perl-build, lib] + + - name: vim + category: core + tags: [c, autotools, editor] + + - name: neovim + category: core + dist: ubuntu + tags: [c, cmake, lua, editor] + + - name: git + category: core + tags: [c, make, vcs] + + - name: rust-ripgrep + category: core + dist: ubuntu + tags: [rust, cargo] + notes: debcargo-style vendored Rust build (binary name is ripgrep) + + - name: make-dfsg + category: core + tier: smoke + tags: [c, autotools, tiny] + notes: the GNU make source package (binary name is make) + + - name: systemd + category: core + dist: ubuntu + tags: [c, meson, init] + + # --- server --------------------------------------------------------------- + - name: nginx + category: server + tags: [c, autoconf, server] + + - name: apache2 + category: server + dist: ubuntu + tags: [c, autotools, server] + + - name: openssh + category: server + tags: [c, autotools, server] + + - name: postfix + category: server + dist: ubuntu + tags: [c, make, mail, server] + + - name: dovecot + category: server + tags: [c, autotools, mail, server] + + - name: samba + category: server + dist: ubuntu + tags: [c, python-waf, server] + + - name: bind9 + category: server + tags: [c, autotools, dns, server] + + - name: haproxy + category: server + dist: ubuntu + tags: [c, make, network, server] + + - name: exim4 + category: server + tags: [c, make, mail, server] + + # --- databases -------------------------------------------------------------- + - name: redis + category: database + tags: [c, make, server] + + - name: sqlite3 + category: database + dist: ubuntu + tags: [c, autotools] + + - name: postgresql-17 + category: database + tags: [c, autotools, server] + notes: long-ish but reliable; a good mid-weight benchmark + + - name: leveldb + category: database + dist: ubuntu + tier: smoke + tags: [cpp, cmake] + + - name: mysql-8.0 + category: database + tier: long + dist: ubuntu + timeout: 240 + tags: [cpp, cmake, server] + notes: Ubuntu-only source name; huge C++ build + + # --- desktop ---------------------------------------------------------------- + - name: gtk4 + category: desktop + tags: [c, meson, gui] + + - name: vlc + category: desktop + dist: ubuntu + tags: [cpp, autotools, gui] + + - name: gimp + category: desktop + tags: [c, meson, gui] + + - name: inkscape + category: desktop + dist: ubuntu + tags: [cpp, cmake, gui] + + - name: mutter + category: desktop + tags: [c, meson, gui] + + - name: pipewire + category: desktop + dist: ubuntu + tags: [c, meson, audio] + + - name: plasma-desktop + category: desktop + tags: [cpp, cmake, kde, gui] + + - name: gnome-control-center + category: desktop + dist: ubuntu + tags: [c, meson, gui] + + - name: ffmpeg + category: desktop + tags: [c, autotools, media] + + # --- toolchains --------------------------------------------------------------- + - name: binutils + category: toolchain + tags: [c, autotools] + + - name: gdb + category: toolchain + dist: ubuntu + tags: [c, autotools] + + - name: cmake + category: toolchain + tier: smoke + tags: [cpp, cmake, bootstrap] + + # --- boot / firmware / weird ------------------------------------------------ + - name: dracut + category: boot-firmware + tier: smoke + tags: [shell, make, initramfs, weird] + notes: unusual make logic; initramfs generator + + - name: opensbi + category: boot-firmware + tier: smoke + tags: [c, make, firmware, weird] + notes: RISC-V firmware, cross-built inside a native build + + - name: grub2 + category: boot-firmware + tags: [c, autotools, bootloader, weird] + notes: multi-platform grub; heavy but dependable + + - name: u-boot + category: boot-firmware + dist: ubuntu + arch: arm64 + cross: true + tags: [c, kconfig, firmware, weird, cross] + notes: cross build for arm64; many board configs + + # --- long: hours each; opt-in via --tier long or --tier all ----------------- + - name: gcc-14 + category: toolchain + dist: ubuntu + tier: long + timeout: 360 + tags: [c, autotools, compiler] + + - name: glibc + category: core + tier: long + timeout: 360 + tags: [c, autotools, libc] + + - name: rustc + category: toolchain + tier: long + timeout: 360 + tags: [rust, compiler] + notes: bootstraps the Rust compiler; the longest single build here + + - name: libreoffice + category: desktop + dist: ubuntu + tier: long + timeout: 480 + tags: [cpp, make, office, gui] + + - name: firefox + category: desktop + tier: long + timeout: 480 + tags: [cpp, rust, browser, gui] + + - name: qt6-base + category: desktop + dist: ubuntu + tier: long + timeout: 360 + tags: [cpp, cmake, gui] + + - name: linux + category: kernel + tier: long + dist: ubuntu + timeout: 480 + tags: [c, kbuild, kernel, weird] + notes: the Ubuntu kernel; multi-hour, many flavours, kbuild quirks apply + + - name: openjdk-21 + category: toolchain + tier: long + dist: ubuntu + timeout: 360 + tags: [java, make, compiler] diff --git a/tests/e2e/run.py b/tests/e2e/run.py new file mode 100755 index 0000000..c046c5c --- /dev/null +++ b/tests/e2e/run.py @@ -0,0 +1,931 @@ +#!/usr/bin/env python3 +"""pkh end-to-end package harness. + +Runs real packages from catalog.yml through the full pkh workflow: + + pkh pull (from the archive, by default) + pkh chlog --rebuild (no-change rebuild entry; EDITOR is stubbed out) + pkh build (.dsc + _source.changes next to the tree) + pkh deb (binary packages in an ephemeral chroot) + +Every stage is logged per package, timed, and checked for its artifacts; +results land in /runs//results.json and results.md. Pulls run in +parallel (network-bound), the build pipeline runs sequentially (one heavy +build at a time, and pkh's shared chroot/apt/ccache caches stay warm). + +Usage: + run.py --list + run.py --dry-run + run.py # smoke + standard tiers + run.py --tier long # the multi-hour builds + run.py --only hello,dracut # exact names + run.py --resume runs//results.json # continue an interrupted run + +See README.md in this directory for the full documentation. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import datetime as dt +import json +import os +import platform +import shlex +import shutil +import signal +import subprocess +import sys +import threading +import time +from dataclasses import dataclass, field +from pathlib import Path + +TIERS = ("smoke", "standard", "long") +DEFAULT_TIERS = ("smoke", "standard") +TIER_TIMEOUT_MIN = {"smoke": 30, "standard": 120, "long": 360} +STAGES = ("pull", "chlog", "build", "deb") +PIPELINE_STAGES = ("chlog", "build", "deb") +TERMINAL_OUTCOMES = ("pass", "xpass", "xfail") +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT_DIR = Path(__file__).resolve().parent + + +class CatalogError(Exception): + """Invalid catalog content or unusable options.""" + + +@dataclass +class Entry: + """One catalog entry: a source package and how to build it.""" + + name: str + category: str = "misc" + tier: str = "standard" + dist: str = "debian" + series: str = "" + arch: str = "" + cross: bool = False + pocket: str = "" + ppa: list[str] = field(default_factory=list) + timeout: float = 0.0 # minutes; 0 = the tier default + xfail: str = "" + notes: str = "" + tags: list[str] = field(default_factory=list) + + def timeout_min(self) -> float: + return self.timeout if self.timeout else TIER_TIMEOUT_MIN.get(self.tier, 120) + + +# --------------------------------------------------------------------------- +# Catalog loading: PyYAML when available, a schema-specific fallback parser +# otherwise (the subset it accepts is documented in catalog.yml). +# --------------------------------------------------------------------------- + + +def _scalar(text: str): + text = text.strip() + if text.startswith("[") and text.endswith("]"): + inner = text[1:-1].strip() + return [v.strip().strip("'\"") for v in inner.split(",") if v.strip()] + if len(text) >= 2 and text[0] == text[-1] and text[0] in "'\"": + return text[1:-1] + if text in ("true", "false"): + return text == "true" + if text.isdigit(): + return int(text) + return text + + +def _parse_mini_yaml(text: str) -> dict: + """Parse the flat 'defaults' mapping + flat 'packages' list subset.""" + doc: dict = {"defaults": {}, "packages": []} + mode = None # None | "defaults" | "packages" + item: dict | None = None + for lineno, raw in enumerate(text.splitlines(), 1): + if not raw.strip() or raw.lstrip().startswith("#"): + continue + indent = len(raw) - len(raw.lstrip(" ")) + stripped = raw.strip() + if indent == 0: + if stripped in ("defaults:", "packages:"): + mode = stripped[:-1] + item = None + continue + raise CatalogError(f"catalog:{lineno}: unexpected top-level '{stripped}'") + if mode is None: + raise CatalogError(f"catalog:{lineno}: content outside a top-level key") + if mode == "defaults": + key, sep, value = stripped.partition(":") + if not sep: + raise CatalogError(f"catalog:{lineno}: expected 'key: value'") + doc["defaults"][key.strip()] = _scalar(value) + else: + if stripped.startswith("- "): + item = {} + doc["packages"].append(item) + stripped = stripped[2:].strip() + if not stripped: + continue + if item is None: + raise CatalogError(f"catalog:{lineno}: list items must start with '- '") + key, sep, value = stripped.partition(":") + if not sep: + raise CatalogError(f"catalog:{lineno}: expected 'key: value'") + item[key.strip()] = _scalar(value) + return doc + + +def load_catalog(path: Path) -> list[Entry]: + text = path.read_text(encoding="utf-8") + use_pyyaml = not os.environ.get("PKH_E2E_NO_PYYAML") + try: + if use_pyyaml: + import yaml # type: ignore import-not-found + + doc = yaml.safe_load(text) + else: + doc = _parse_mini_yaml(text) + except CatalogError: + raise + except Exception as exc: + if use_pyyaml: + raise CatalogError(f"cannot parse {path}: {exc}") from exc + raise + + defaults = doc.get("defaults") or {} + raw_entries = doc.get("packages") or [] + if not raw_entries: + raise CatalogError(f"{path}: no packages listed") + + entries: list[Entry] = [] + seen: set[str] = set() + for i, raw in enumerate(raw_entries): + name = str(raw.get("name", "")).strip() + if not name: + raise CatalogError(f"{path}: package #{i + 1} has no name") + if name in seen: + raise CatalogError(f"{path}: duplicate package '{name}'") + seen.add(name) + + entry = Entry( + name=name, + category=str(raw.get("category", "misc")), + tier=str(raw.get("tier", defaults.get("tier", "standard"))), + dist=str(raw.get("dist", defaults.get("dist", "debian"))), + series=str(raw.get("series", "")), + arch=str(raw.get("arch", "")), + cross=bool(raw.get("cross", False)), + pocket=str(raw.get("pocket", "")), + ppa=[str(p) for p in (raw.get("ppa") or [])], + timeout=float(raw.get("timeout", 0)), + xfail=str(raw.get("xfail", "")), + notes=str(raw.get("notes", "")), + tags=[str(t) for t in (raw.get("tags") or [])], + ) + if entry.tier not in TIERS: + raise CatalogError(f"{path}: {name}: unknown tier '{entry.tier}'") + if entry.dist not in ("debian", "ubuntu"): + raise CatalogError(f"{path}: {name}: unknown dist '{entry.dist}'") + if entry.cross and not entry.arch: + raise CatalogError(f"{path}: {name}: cross: true needs an arch") + entries.append(entry) + return entries + + +# --------------------------------------------------------------------------- +# Selection and rendering +# --------------------------------------------------------------------------- + + +def select_entries( + entries: list[Entry], args: argparse.Namespace +) -> tuple[list[Entry], list[str]]: + """Apply the CLI filters; returns the selection and the filtered-out names.""" + tiers = ( + list(TIERS) if args.tier == "all" else [t.strip() for t in args.tier.split(",")] + ) + for tier in tiers: + if tier not in TIERS: + raise CatalogError(f"unknown tier '{tier}' (expected: {', '.join(TIERS)})") + + selected = [e for e in entries if e.tier in tiers] + dropped = [e.name for e in entries if e.tier not in tiers] + + if args.category: + wanted = {c.strip() for c in args.category.split(",") if c.strip()} + dropped += [e.name for e in selected if e.category not in wanted] + selected = [e for e in selected if e.category in wanted] + if args.tag: + wanted = {t.strip() for t in args.tag.split(",") if t.strip()} + dropped += [e.name for e in selected if not wanted.intersection(e.tags)] + selected = [e for e in selected if wanted.intersection(e.tags)] + if args.only: + names = {n.strip() for n in args.only.split(",") if n.strip()} + unknown = sorted(names - {e.name for e in entries}) + if unknown: + raise CatalogError(f"--only: not in the catalog: {', '.join(unknown)}") + dropped += [e.name for e in selected if e.name not in names] + selected = [e for e in selected if e.name in names] + if args.skip: + names = {n.strip() for n in args.skip.split(",") if n.strip()} + dropped += [e.name for e in selected if e.name in names] + selected = [e for e in selected if e.name not in names] + return selected, dropped + + +def fmt_duration(seconds: float | None) -> str: + if seconds is None: + return "-" + seconds = int(seconds) + if seconds < 60: + return f"{seconds}s" + if seconds < 3600: + return f"{seconds // 60}m{seconds % 60:02d}s" + return f"{seconds // 3600}h{(seconds % 3600) // 60:02d}m" + + +def render_list(entries: list[Entry]) -> str: + lines = [ + f"{'PACKAGE':<20} {'TIER':<9} {'DIST':<7} {'CATEGORY':<14} " + f"{'TAGS':<30} NOTES" + ] + for e in entries: + extra = [] + if e.cross: + extra.append(f"cross {e.arch}") + if e.series: + extra.append(f"series {e.series}") + note = " ".join(extra + [e.xfail or e.notes]) + lines.append( + f"{e.name:<20} {e.tier:<9} {e.dist:<7} {e.category:<14} " + f"{','.join(e.tags):<30} {note}" + ) + return "\n".join(lines) + f"\n\n{len(entries)} package(s)\n" + + +# --------------------------------------------------------------------------- +# Command construction and child-process plumbing +# --------------------------------------------------------------------------- + + +def stage_command(args: argparse.Namespace, entry: Entry, stage: str, pkh: Path) -> list[str]: + cmd = [str(pkh), stage] + if stage == "pull": + cmd += [entry.name, "-d", args.dist or entry.dist] + if args.series or entry.series: + cmd += ["-s", args.series or entry.series] + if args.pocket or entry.pocket: + cmd += ["-p", args.pocket or entry.pocket] + if not args.git: + cmd += ["--archive"] + elif stage == "chlog": + cmd += ["--rebuild"] + if args.series or entry.series: + cmd += ["-s", args.series or entry.series] + elif stage == "build": + cmd += ["--verbose"] + elif stage == "deb": + cmd += ["--verbose"] + if args.series or entry.series: + cmd += ["-s", args.series or entry.series] + arch = args.arch or entry.arch + if arch: + cmd += ["-a", arch] + if entry.cross or args.cross: + cmd += ["--cross"] + if args.pocket or entry.pocket: + cmd += ["-p", args.pocket or entry.pocket] + for ppa in entry.ppa + args.ppa: + cmd += ["--ppa", ppa] + if args.jobs: + cmd += ["-j", str(args.jobs)] + if args.no_ccache: + cmd += ["--no-ccache"] + return cmd + + +def child_env() -> dict: + """Environment for pkh children: no editor, a deterministic identity.""" + env = dict(os.environ) + env["EDITOR"] = "true" # chlog launches it after generating; nothing to edit + env.setdefault("DEBEMAIL", "pkh-e2e@invalid") + env.setdefault("DEBFULLNAME", "pkh e2e") + return env + + +def run_stage( + cmd: list[str], cwd: Path, log_path: Path, timeout_min: float +) -> tuple[str, float]: + """Run one stage, logging to log_path; returns (status, seconds). + + Status is 'ok', 'fail' or 'timeout'. The whole process group is killed + on timeout: builds fan out into make/compiler children, and killing only + the direct child would leave them running. On Ctrl+C the group gets a + graceful SIGTERM first, mirroring pkh's own interrupt cleanup. + """ + with log_path.open("ab") as log: + log.write(f"\n$ {shlex.join(cmd)}\n(cwd: {cwd})\n".encode()) + log.flush() + start = time.monotonic() + proc = subprocess.Popen( + cmd, + cwd=cwd, + env=child_env(), + stdin=subprocess.DEVNULL, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + try: + proc.wait(timeout=timeout_min * 60) + except subprocess.TimeoutExpired: + _kill_group(proc, signal.SIGKILL) + return "timeout", time.monotonic() - start + except KeyboardInterrupt: + # A graceful stop first: SIGTERM lets pkh run its own interrupt + # cleanup; a child ignoring it is killed after a grace period. + _kill_group(proc, signal.SIGTERM, wait=10) + _kill_group(proc, signal.SIGKILL) + raise + return ("ok" if proc.returncode == 0 else "fail"), time.monotonic() - start + + +def _kill_group( + proc: subprocess.Popen, sig: signal.Signals, wait: float | None = None +) -> None: + try: + os.killpg(os.getpgid(proc.pid), sig) + except ProcessLookupError: + pass + try: + proc.wait(timeout=wait) + except subprocess.TimeoutExpired: + pass + + +# --------------------------------------------------------------------------- +# The harness proper +# --------------------------------------------------------------------------- + + +class Harness: + def __init__(self, args: argparse.Namespace, entries: list[Entry], pkh: Path): + self.args = args + self.entries = entries + self.pkh = pkh + self.stages = [s.strip() for s in args.stages.split(",") if s.strip()] + self.records: dict[str, dict] = {} + self.pull_results: dict[str, tuple[str, float]] = {} + self.lock = threading.Lock() + self.run_dir = self._open_run_dir() + self.results_path = self.run_dir / "results.json" + (self.run_dir / "logs").mkdir(parents=True, exist_ok=True) + + # -- layout ------------------------------------------------------------- + + def _open_run_dir(self) -> Path: + if self.args.resume: + try: + data = json.loads(Path(self.args.resume).read_text()) + run_dir = Path(data["run_dir"]) + except (OSError, KeyError, json.JSONDecodeError) as exc: + raise CatalogError(f"--resume: cannot read {self.args.resume}: {exc}") + if not run_dir.is_dir(): + raise CatalogError(f"--resume: missing run directory {run_dir}") + for record in data.get("entries", []): + self.records[record["name"]] = record + return run_dir + stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S") + return SCRIPT_DIR / "runs" / f"{stamp}-{os.getpid()}" + + def log_path(self, name: str) -> Path: + return self.run_dir / "logs" / f"{name}.log" + + def workroot(self, entry: Entry) -> Path: + # pkh pull (run from the run root) puts the tree at + # .///: this directory is both the tree's parent and + # where pkh build / pkh deb drop their artifacts. + return self.run_dir / entry.name + + def tree(self, entry: Entry) -> Path: + return self.workroot(entry) / entry.name + + # -- stages --------------------------------------------------------------- + + def do_pull(self, entry: Entry) -> tuple[str, float]: + # Pull from the run root: pkh pull (cwd-less) extracts to + # ./// under its cwd, so every package still gets its own + # subtree and parallel pulls never collide. A failed pull leaves a + # partial tree that pkh pull would refuse to overwrite and that a + # later --resume must not trust: wipe it before retrying and after + # the last attempt. + for attempt in range(self.args.pull_retries + 1): + if attempt: + shutil.rmtree(self.workroot(entry), ignore_errors=True) + status, seconds = run_stage( + stage_command(self.args, entry, "pull", self.pkh), + self.run_dir, + self.log_path(entry.name), + entry.timeout_min(), + ) + if status == "ok": + return status, seconds + shutil.rmtree(self.workroot(entry), ignore_errors=True) + return status, seconds + + def do_chlog(self, entry: Entry) -> tuple[str, float]: + return run_stage( + stage_command(self.args, entry, "chlog", self.pkh), + self.tree(entry), + self.log_path(entry.name), + entry.timeout_min(), + ) + + def do_build(self, entry: Entry) -> tuple[tuple[str, float], list[str]]: + record = run_stage( + stage_command(self.args, entry, "build", self.pkh), + self.tree(entry), + self.log_path(entry.name), + entry.timeout_min(), + ) + artifacts = self.build_artifacts(entry) + return self._check_artifacts(entry, record, artifacts, "dsc/changes"), artifacts + + def do_deb(self, entry: Entry) -> tuple[tuple[str, float], list[str]]: + record = run_stage( + stage_command(self.args, entry, "deb", self.pkh), + self.tree(entry), + self.log_path(entry.name), + entry.timeout_min(), + ) + debs = self.deb_artifacts(entry) + return self._check_artifacts(entry, record, debs, ".deb"), debs + + def build_artifacts(self, entry: Entry) -> list[str]: + return sorted( + p.name + for pattern in ("*.dsc", "*_source.changes") + for p in self.workroot(entry).glob(pattern) + ) + + def deb_artifacts(self, entry: Entry) -> list[str]: + return sorted(p.name for p in self.workroot(entry).glob("*.deb")) + + def _check_artifacts( + self, entry: Entry, record: tuple[str, float], artifacts: list[str], what: str + ) -> tuple[str, float]: + """A green stage with no artifacts is a failure: log it and flip.""" + if record[0] == "ok" and not artifacts: + with self.log_path(entry.name).open("ab") as log: + log.write( + f"\npkh-e2e: stage reported success but produced no {what}\n".encode() + ) + return ("fail", record[1]) + return record + + def changelog_version(self, entry: Entry) -> str: + try: + out = subprocess.run( + ["dpkg-parsechangelog", "-S", "Version"], + cwd=self.tree(entry), + capture_output=True, + text=True, + check=True, + ) + return out.stdout.strip() + except Exception: + return "" + + # -- per-entry pipeline ------------------------------------------------------ + + def run_entry(self, entry: Entry) -> dict: + """Full pipeline for one entry; never raises.""" + # A resumed run keeps what the previous record already achieved + previous = self.records.get(entry.name) + record = { + "name": entry.name, + "category": entry.category, + "tier": entry.tier, + "dist": self.args.dist or entry.dist, + "series": self.args.series or entry.series, + "arch": self.args.arch or entry.arch or platform.machine(), + "cross": entry.cross or self.args.cross, + "tags": entry.tags, + "xfail": entry.xfail, + "notes": entry.notes, + "outcome": "fail", + "version": "", + "stages": {}, + "artifacts": [], + "total_seconds": 0.0, + "log": str(self.log_path(entry.name).relative_to(self.run_dir)), + } + started = time.monotonic() + try: + self._run_entry(record, entry, previous) + finally: + record["total_seconds"] = time.monotonic() - started + if entry.xfail: + record["outcome"] = "xfail" if record["outcome"] == "fail" else "xpass" + with self.lock: + self.records[entry.name] = record + self.flush() + color = {"pass": 32, "xfail": 33, "xpass": 33}.get(record["outcome"], 31) + print( + f"[{entry.name}] \033[{color}m{record['outcome']}\033[0m " + f"({fmt_duration(record['total_seconds'])})" + + (f" — {record['error']}" if record.get("error") else "") + ) + return record + + def _run_entry(self, record: dict, entry: Entry, previous: dict | None) -> None: + # Stages the resumed record already completed are reused rather than + # re-run: a resumed chlog must not bump the version a second time. + # Reuse only while the build selection is unchanged. + reusable = previous is not None and all( + previous.get(key) == record.get(key) + for key in ("dist", "series", "arch", "cross") + ) + prev_stages = previous.get("stages", {}) if previous else {} + + if "pull" in self.stages: + if (self.tree(entry) / "debian").is_dir(): + # Resumed run: the tree survived, keep it (pkh pull refuses + # existing directories anyway). + record["stages"]["pull"] = {"status": "ok", "seconds": 0.0, + "cached": True} + print(f"[{entry.name}] pull ok (tree already present)") + else: + status, seconds = self.pull_results.get(entry.name, ("fail", 0.0)) + record["stages"]["pull"] = {"status": status, "seconds": seconds} + if status != "ok": + record["error"] = f"pull {status}" + return + # Already reported by the pull phase; no duplicate line. + + for stage in PIPELINE_STAGES: + if stage not in self.stages: + continue + prior = prev_stages.get(stage) if reusable else None + if prior and prior.get("status") == "ok": + record["stages"][stage] = {**prior, "reused": True} + if stage == "chlog": + record["version"] = previous.get("version", "") + elif stage == "build": + record["artifacts"] += self.build_artifacts(entry) + elif stage == "deb": + record["artifacts"] += self.deb_artifacts(entry) + print(f"[{entry.name}] {stage} ok (reused)") + continue + if not (self.tree(entry) / "debian").is_dir(): + record["error"] = ( + f"no source tree for {stage}; include the pull stage or " + "resume the original run directory" + ) + return + + if stage == "chlog": + status, seconds = self.do_chlog(entry) + record["stages"]["chlog"] = {"status": status, "seconds": seconds} + if status != "ok": + record["error"] = f"chlog {status}" + return + record["version"] = self.changelog_version(entry) + print( + f"[{entry.name}] chlog ok ({fmt_duration(seconds)})" + + (f" -> {record['version']}" if record["version"] else "") + ) + elif stage == "build": + (status, seconds), artifacts = self.do_build(entry) + record["stages"]["build"] = {"status": status, "seconds": seconds} + record["artifacts"] += artifacts + if status != "ok": + record["error"] = f"build {status}" + return + print(f"[{entry.name}] build ok ({fmt_duration(seconds)})") + elif stage == "deb": + (status, seconds), debs = self.do_deb(entry) + record["stages"]["deb"] = {"status": status, "seconds": seconds} + record["artifacts"] += debs + if status != "ok": + record["error"] = f"deb {status}" + return + print( + f"[{entry.name}] deb ok ({fmt_duration(seconds)}), " + f"{len(debs)} binary package(s)" + ) + + record["outcome"] = "pass" + + # -- run orchestration --------------------------------------------------------- + + def preflight(self) -> None: + version = subprocess.run( + [str(self.pkh), "--version"], capture_output=True, text=True + ) + print(f"[pkh-e2e] pkh: {self.pkh} ({version.stdout.strip() or '?'})") + if "deb" in self.stages: + missing = [ + tool + for tool in ("mmdebstrap", "unshare", "sudo") + if shutil.which(tool) is None + ] + if missing: + print(f"[pkh-e2e] WARNING: deb needs: {', '.join(missing)}") + if subprocess.run(["sudo", "-n", "true"], capture_output=True).returncode: + print( + "[pkh-e2e] WARNING: passwordless sudo unavailable; " + "ephemeral chroot builds will fail" + ) + free_g = shutil.disk_usage(self.run_dir).free / 2**30 + if free_g < 20: + print(f"[pkh-e2e] WARNING: only {free_g:.0f}G free on {self.run_dir}") + print( + f"[pkh-e2e] {len(self.entries)} package(s), stages: " + f"{'+'.join(self.stages)} -> {self.run_dir}" + ) + + def resume_skip(self, entry: Entry) -> bool: + previous = self.records.get(entry.name) + if not previous or previous.get("outcome") not in TERMINAL_OUTCOMES: + return False + return all(s in previous.get("stages", {}) for s in self.stages) + + def pull_phase(self) -> None: + if "pull" not in self.stages: + return + todo = [ + e + for e in self.entries + if not self.resume_skip(e) and not (self.tree(e) / "debian").is_dir() + ] + if not todo: + return + print( + f"[pkh-e2e] pulling {len(todo)} package(s) " + f"({self.args.pull_jobs} at a time)" + ) + with concurrent.futures.ThreadPoolExecutor( + max_workers=self.args.pull_jobs + ) as pool: + futures = {pool.submit(self.do_pull, e): e for e in todo} + for future in concurrent.futures.as_completed(futures): + entry = futures[future] + try: + status, seconds = future.result() + except Exception as exc: # keep the run going on worker errors + status, seconds = "fail", 0.0 + print(f"[{entry.name}] pull crashed: {exc}") + self.pull_results[entry.name] = (status, seconds) + print( + f"[{entry.name}] pull {'ok' if status == 'ok' else status} " + f"({fmt_duration(seconds)})" + ) + + def run(self) -> int: + self.preflight() + self.flush() + + self.pull_phase() + + failures = 0 + for entry in self.entries: + if self.resume_skip(entry): + print(f"[{entry.name}] already {self.records[entry.name]['outcome']}," + " skipping") + continue + if self.args.fail_fast and failures: + print(f"[{entry.name}] skipped (fail-fast)") + continue + record = self.run_entry(entry) + if record["outcome"] in ("fail", "timeout"): + failures += 1 + + self.write_report() + self.print_summary() + return self.exit_code(failures) + + def print_summary(self) -> None: + rows = sorted(self.records.values(), key=lambda r: r["name"]) + print(f"\n=== summary ({len(rows)} recorded) ===") + print( + f"{'PACKAGE':<20} {'TIER':<9} {'RESULT':<8} " + f"{'PULL':>7} {'CHLOG':>7} {'BUILD':>8} {'DEB':>8} {'TOTAL':>8} VERSION" + ) + for r in rows: + def sec(stage): + data = r["stages"].get(stage) + return fmt_duration(data["seconds"]) if data else "-" + + print( + f"{r['name']:<20} {r['tier']:<9} {r['outcome']:<8} " + f"{sec('pull'):>7} {sec('chlog'):>7} {sec('build'):>8} " + f"{sec('deb'):>8} {fmt_duration(r['total_seconds']):>8} " + f"{r['version']}" + ) + tally: dict[str, int] = {} + for r in rows: + tally[r["outcome"]] = tally.get(r["outcome"], 0) + 1 + print("\n" + ", ".join(f"{v} {k}" for k, v in sorted(tally.items()))) + + def exit_code(self, failures: int) -> int: + if failures: + return 1 + if self.args.strict and any( + r["outcome"] == "xpass" for r in self.records.values() + ): + return 1 + return 0 + + # -- persistence --------------------------------------------------------------- + + def flush(self) -> None: + """Persist results progressively: a killed run keeps its records.""" + if not hasattr(self, "_started"): + self._started = dt.datetime.now().astimezone().isoformat() + meta = { + "run_dir": str(self.run_dir), + "pkh": str(self.pkh), + "filters": { + "tier": self.args.tier, + "category": self.args.category, + "tag": self.args.tag, + "only": self.args.only, + "skip": self.args.skip, + "stages": self.stages, + }, + "host": { + "kernel": platform.release(), + "arch": platform.machine(), + "python": platform.python_version(), + }, + "started": self._started, + "updated": dt.datetime.now().astimezone().isoformat(), + "entries": sorted(self.records.values(), key=lambda r: r["name"]), + } + tmp = self.results_path.with_suffix(".json.tmp") + tmp.write_text(json.dumps(meta, indent=2) + "\n") + tmp.replace(self.results_path) + + def write_report(self) -> None: + rows = sorted(self.records.values(), key=lambda r: (r["outcome"], r["name"])) + lines = [ + f"# pkh e2e run — {self.run_dir.name}", + "", + f"- pkh: `{self.pkh}`", + f"- started: {getattr(self, '_started', '?')}", + f"- stages: {'+'.join(self.stages)}", + "", + "| package | tier | result | pull | chlog | build | deb | total |" + " version |", + "|---|---|---|---|---|---|---|---|---|", + ] + for r in rows: + def sec(stage): + data = r["stages"].get(stage) + return fmt_duration(data["seconds"]) if data else "-" + + lines.append( + f"| {r['name']} | {r['tier']} | {r['outcome']} | {sec('pull')} | " + f"{sec('chlog')} | {sec('build')} | {sec('deb')} | " + f"{fmt_duration(r['total_seconds'])} | {r['version']} |" + ) + (self.run_dir / "results.md").write_text("\n".join(lines) + "\n") + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--catalog", type=Path, default=SCRIPT_DIR / "catalog.yml", + help="catalog file (default: catalog.yml)") + parser.add_argument("--tier", default=",".join(DEFAULT_TIERS), + help="tiers to run, comma-separated: smoke, standard, long, " + "or 'all' (default: smoke,standard)") + parser.add_argument("--category", default="", + help="comma-separated categories to keep") + parser.add_argument("--tag", default="", + help="comma-separated tags; keep packages carrying any") + parser.add_argument("--only", default="", + help="comma-separated exact package names to run") + parser.add_argument("--skip", default="", + help="comma-separated exact package names to skip") + parser.add_argument("--stages", default=",".join(STAGES), + help=f"stages to run (default: {','.join(STAGES)})") + parser.add_argument("--pkh", default="", + help="pkh binary to exercise (default: the repo build, " + "then PATH)") + parser.add_argument("--build-pkh", action="store_true", + help="run 'cargo build --release' before resolving pkh") + parser.add_argument("--git", action="store_true", + help="pull through the package VCS instead of the archive") + parser.add_argument("--dist", default="", + help="override the catalog default dist (per-entry dist wins)") + parser.add_argument("--series", default="", + help="override the series (per-entry series wins)") + parser.add_argument("--arch", default="", + help="force the 'pkh deb -a' architecture on every entry") + parser.add_argument("--cross", action="store_true", + help="pass --cross to every deb build") + parser.add_argument("--pocket", default="", + help="distribution pocket passed to pull and deb") + parser.add_argument("--ppa", action="append", default=[], + help="PPA passed to 'pkh deb --ppa' (repeatable)") + parser.add_argument("--jobs", type=int, default=0, + help="pass -j N to 'pkh deb' (default: pkh chooses)") + parser.add_argument("--no-ccache", action="store_true", + help="pass --no-ccache to 'pkh deb'") + parser.add_argument("--pull-jobs", type=int, default=4, + help="parallel pulls (default: 4)") + parser.add_argument("--pull-retries", type=int, default=1, + help="retries per failed pull (default: 1)") + parser.add_argument("--fail-fast", action="store_true", + help="stop the pipeline at the first failure") + parser.add_argument("--strict", action="store_true", + help="also fail the run on xpass") + parser.add_argument("--resume", default="", + help="results.json of a previous run: reuse its run " + "directory and skip finished packages") + parser.add_argument("--list", action="store_true", + help="print the selected packages and exit") + parser.add_argument("--dry-run", action="store_true", + help="print the plan and the exact commands, run nothing") + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + try: + entries = load_catalog(args.catalog) + selected, dropped = select_entries(entries, args) + + stages = [s.strip() for s in args.stages.split(",") if s.strip()] + for stage in stages: + if stage not in STAGES: + raise CatalogError( + f"unknown stage '{stage}' (expected: {', '.join(STAGES)})" + ) + if args.pull_jobs < 1: + raise CatalogError("--pull-jobs must be at least 1") + + if args.list: + print(render_list(selected)) + if dropped: + print(f"(filtered out: {len(dropped)})") + return 0 + if not selected: + print("nothing selected", file=sys.stderr) + return 2 + + pkh = resolve_pkh(args) + if args.dry_run: + print(f"pkh: {pkh}") + print(f"selected {len(selected)} package(s), stages: {args.stages}") + for entry in selected: + where = f" {entry.series}" if entry.series else "" + print(f"\n[{entry.name}] {entry.tier}/{entry.category} " + f"({entry.dist}{where})") + for stage in stages: + cmd = stage_command(args, entry, stage, pkh) + print(f" {shlex.join(cmd)}" + f" # timeout {entry.timeout_min():.0f}m") + return 0 + + harness = Harness(args, selected, pkh) + return harness.run() + except CatalogError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + except KeyboardInterrupt: + print("\ninterrupted", file=sys.stderr) + return 130 + + +def resolve_pkh(args: argparse.Namespace) -> Path: + if args.pkh: + path = Path(args.pkh).resolve() + if not path.is_file(): + raise CatalogError(f"--pkh: no such file: {path}") + return path + if args.build_pkh: + subprocess.run(["cargo", "build", "--release"], cwd=REPO_ROOT, check=True) + for candidate in (REPO_ROOT / "target/release/pkh", REPO_ROOT / "target/debug/pkh"): + if candidate.is_file(): + return candidate + found = shutil.which("pkh") + if not found: + raise CatalogError( + "no pkh binary: pass --pkh, use --build-pkh, or install pkh on PATH" + ) + return Path(found) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))