test: add the e2e package harness

Exercises pkh the way it is used in anger: 63 real source packages (from hello and micro to dracut, grub2 and the kernel) run through pull -> chlog --rebuild -> build -> deb, each stage timed, logged and checked for its artifacts, with the catalog alternating between debian and ubuntu within every category. run.py keeps runs self-contained (per-package logs, progressive results.json, a markdown summary), pulls in parallel while builds run sequentially to keep pkh's shared chroot/apt/ccache caches warm, resumes interrupted runs reusing the stages that already succeeded, and marks known-broken entries as xfail with a reason. The first catalog run caught four pkh defects within an hour - the unstable/sid suite alias, the bz2 origs, the hard link extraction and the wrapper unwrapping - which is exactly what it is for.
2026-10-01 22:24:07 +02:00
parent a4be046aaa
commit 87f5b31602
5 changed files with 1411 additions and 0 deletions
@@ -0,0 +1,375 @@
+# pkh end-to-end test catalog
+#
+# One entry per source package, exercised by run.py as:
+#   pkh pull -> pkh chlog --rebuild -> pkh build -> pkh deb
+#
+# The catalog deliberately alternates between debian and ubuntu within
+# every category, so a default run exercises both archives, series
+# namings and chroots.
+#
+# The file is plain YAML but is also parsed by a small built-in parser when
+# PyYAML is unavailable: keep to the subset it supports — full-line comments,
+# 'key: value' scalars, inline [a, b] lists, no nested mappings, no block
+# scalars, no colons inside values.
+#
+# Fields (all optional except 'name'):
+#   category   free grouping label (smoke, server, desktop, ...)   [defaults.tier]
+#   tier       smoke | standard | long — 'long' is only run when asked for
+#   dist       debian | ubuntu                                     [defaults.dist]
+#   series     pin a series (default: the dist development series)
+#   arch       pass '-a ARCH' to 'pkh deb'
+#   cross      true -> also pass '--cross' (needs 'arch')          [false]
+#   pocket     pass '-p POCKET' to pull/deb
+#   ppa        inline list of PPAs passed to 'pkh deb --ppa'
+#   timeout    per-stage budget in minutes                         [tier default]
+#   xfail      known-broken marker: the reason it is expected to fail
+#   notes      free text, shown by --list and in reports
+
+defaults:
+  dist: debian
+  tier: standard
+
+packages:
+  # --- smoke: small, fast, reliable; the first line of defense -------------
+  - name: hello
+    category: smoke
+    tier: smoke
+    tags: [gnu, autotools, tiny]
+    notes: the canonical packaging smoke test
+
+  - name: micro
+    category: smoke
+    tier: smoke
+    tags: [go, editor, tiny]
+    notes: Go build, golang build-dependencies resolved offline
+
+  - name: jq
+    category: smoke
+    tier: smoke
+    tags: [c, autotools, tiny]
+
+  - name: htop
+    category: smoke
+    dist: ubuntu
+    tier: smoke
+    tags: [c, autotools, tiny]
+
+  - name: tmux
+    category: smoke
+    tier: smoke
+    tags: [c, autotools, tiny]
+
+  - name: tree
+    category: smoke
+    dist: ubuntu
+    tier: smoke
+    tags: [c, make, tiny]
+
+  - name: figlet
+    category: smoke
+    dist: ubuntu
+    tier: smoke
+    tags: [c, make, tiny]
+
+  - name: dash
+    category: smoke
+    tier: smoke
+    tags: [c, autotools, shell]
+
+  - name: file
+    category: smoke
+    tier: smoke
+    tags: [c, autotools]
+
+  - name: zlib
+    category: smoke
+    dist: ubuntu
+    tier: smoke
+    tags: [c, autotools, lib]
+
+  - name: xz-utils
+    category: smoke
+    tier: smoke
+    tags: [c, autotools]
+    xfail: sid carries the 5.8.1-1+deb13u1 trixie upload whose packaging fails at dh_installdocs (doxygen api docs missing) - package-side, under triage
+
+  - name: busybox
+    category: smoke
+    dist: ubuntu
+    tier: smoke
+    tags: [c, kconfig, static]
+
+  - name: dnsmasq
+    category: smoke
+    tier: smoke
+    tags: [c, make, network]
+
+  - name: memcached
+    category: smoke
+    dist: ubuntu
+    tier: smoke
+    tags: [c, autotools, server]
+
+  - name: kbuild
+    category: smoke
+    tier: smoke
+    tags: [c, kbuild, weird]
+    notes: the virtualbox build system; exercises the kbuild log classifier
+
+  # --- core -----------------------------------------------------------------
+  - name: bash
+    category: core
+    tags: [c, autotools, shell]
+
+  - name: coreutils
+    category: core
+    dist: ubuntu
+    tags: [c, autotools]
+    xfail: pkh deb builds as root in the chroot and coreutils configure refuses root (needs a non-root build user, as Launchpad buildds use)
+
+  - name: util-linux
+    category: core
+    tags: [c, autotools]
+
+  - name: openssl
+    category: core
+    dist: ubuntu
+    tags: [c, perl-build, lib]
+
+  - name: vim
+    category: core
+    tags: [c, autotools, editor]
+
+  - name: neovim
+    category: core
+    dist: ubuntu
+    tags: [c, cmake, lua, editor]
+
+  - name: git
+    category: core
+    tags: [c, make, vcs]
+
+  - name: rust-ripgrep
+    category: core
+    dist: ubuntu
+    tags: [rust, cargo]
+    notes: debcargo-style vendored Rust build (binary name is ripgrep)
+
+  - name: make-dfsg
+    category: core
+    tier: smoke
+    tags: [c, autotools, tiny]
+    notes: the GNU make source package (binary name is make)
+
+  - name: systemd
+    category: core
+    dist: ubuntu
+    tags: [c, meson, init]
+
+  # --- server ---------------------------------------------------------------
+  - name: nginx
+    category: server
+    tags: [c, autoconf, server]
+
+  - name: apache2
+    category: server
+    dist: ubuntu
+    tags: [c, autotools, server]
+
+  - name: openssh
+    category: server
+    tags: [c, autotools, server]
+
+  - name: postfix
+    category: server
+    dist: ubuntu
+    tags: [c, make, mail, server]
+
+  - name: dovecot
+    category: server
+    tags: [c, autotools, mail, server]
+
+  - name: samba
+    category: server
+    dist: ubuntu
+    tags: [c, python-waf, server]
+
+  - name: bind9
+    category: server
+    tags: [c, autotools, dns, server]
+
+  - name: haproxy
+    category: server
+    dist: ubuntu
+    tags: [c, make, network, server]
+
+  - name: exim4
+    category: server
+    tags: [c, make, mail, server]
+
+  # --- databases --------------------------------------------------------------
+  - name: redis
+    category: database
+    tags: [c, make, server]
+
+  - name: sqlite3
+    category: database
+    dist: ubuntu
+    tags: [c, autotools]
+
+  - name: postgresql-17
+    category: database
+    tags: [c, autotools, server]
+    notes: long-ish but reliable; a good mid-weight benchmark
+
+  - name: leveldb
+    category: database
+    dist: ubuntu
+    tier: smoke
+    tags: [cpp, cmake]
+
+  - name: mysql-8.0
+    category: database
+    tier: long
+    dist: ubuntu
+    timeout: 240
+    tags: [cpp, cmake, server]
+    notes: Ubuntu-only source name; huge C++ build
+
+  # --- desktop ----------------------------------------------------------------
+  - name: gtk4
+    category: desktop
+    tags: [c, meson, gui]
+
+  - name: vlc
+    category: desktop
+    dist: ubuntu
+    tags: [cpp, autotools, gui]
+
+  - name: gimp
+    category: desktop
+    tags: [c, meson, gui]
+
+  - name: inkscape
+    category: desktop
+    dist: ubuntu
+    tags: [cpp, cmake, gui]
+
+  - name: mutter
+    category: desktop
+    tags: [c, meson, gui]
+
+  - name: pipewire
+    category: desktop
+    dist: ubuntu
+    tags: [c, meson, audio]
+
+  - name: plasma-desktop
+    category: desktop
+    tags: [cpp, cmake, kde, gui]
+
+  - name: gnome-control-center
+    category: desktop
+    dist: ubuntu
+    tags: [c, meson, gui]
+
+  - name: ffmpeg
+    category: desktop
+    tags: [c, autotools, media]
+
+  # --- toolchains ---------------------------------------------------------------
+  - name: binutils
+    category: toolchain
+    tags: [c, autotools]
+
+  - name: gdb
+    category: toolchain
+    dist: ubuntu
+    tags: [c, autotools]
+
+  - name: cmake
+    category: toolchain
+    tier: smoke
+    tags: [cpp, cmake, bootstrap]
+
+  # --- boot / firmware / weird ------------------------------------------------
+  - name: dracut
+    category: boot-firmware
+    tier: smoke
+    tags: [shell, make, initramfs, weird]
+    notes: unusual make logic; initramfs generator
+
+  - name: opensbi
+    category: boot-firmware
+    tier: smoke
+    tags: [c, make, firmware, weird]
+    notes: RISC-V firmware, cross-built inside a native build
+
+  - name: grub2
+    category: boot-firmware
+    tags: [c, autotools, bootloader, weird]
+    notes: multi-platform grub; heavy but dependable
+
+  - name: u-boot
+    category: boot-firmware
+    dist: ubuntu
+    arch: arm64
+    cross: true
+    tags: [c, kconfig, firmware, weird, cross]
+    notes: cross build for arm64; many board configs
+
+  # --- long: hours each; opt-in via --tier long or --tier all -----------------
+  - name: gcc-14
+    category: toolchain
+    dist: ubuntu
+    tier: long
+    timeout: 360
+    tags: [c, autotools, compiler]
+
+  - name: glibc
+    category: core
+    tier: long
+    timeout: 360
+    tags: [c, autotools, libc]
+
+  - name: rustc
+    category: toolchain
+    tier: long
+    timeout: 360
+    tags: [rust, compiler]
+    notes: bootstraps the Rust compiler; the longest single build here
+
+  - name: libreoffice
+    category: desktop
+    dist: ubuntu
+    tier: long
+    timeout: 480
+    tags: [cpp, make, office, gui]
+
+  - name: firefox
+    category: desktop
+    tier: long
+    timeout: 480
+    tags: [cpp, rust, browser, gui]
+
+  - name: qt6-base
+    category: desktop
+    dist: ubuntu
+    tier: long
+    timeout: 360
+    tags: [cpp, cmake, gui]
+
+  - name: linux
+    category: kernel
+    tier: long
+    dist: ubuntu
+    timeout: 480
+    tags: [c, kbuild, kernel, weird]
+    notes: the Ubuntu kernel; multi-hour, many flavours, kbuild quirks apply
+
+  - name: openjdk-21
+    category: toolchain
+    tier: long
+    dist: ubuntu
+    timeout: 360
+    tags: [java, make, compiler]