From 87f5b31602dbb01513add66dc4a63922fcfb63cb Mon Sep 17 00:00:00 2001 From: Valentin Haudiquet Date: Thu, 1 Oct 2026 22:24:07 +0200 Subject: [PATCH] test: add the e2e package harness Exercises pkh the way it is used in anger: 63 real source packages (from hello and micro to dracut, grub2 and the kernel) run through pull -> chlog --rebuild -> build -> deb, each stage timed, logged and checked for its artifacts, with the catalog alternating between debian and ubuntu within every category. run.py keeps runs self-contained (per-package logs, progressive results.json, a markdown summary), pulls in parallel while builds run sequentially to keep pkh's shared chroot/apt/ccache caches warm, resumes interrupted runs reusing the stages that already succeeded, and marks known-broken entries as xfail with a reason. The first catalog run caught four pkh defects within an hour - the unstable/sid suite alias, the bz2 origs, the hard link extraction and the wrapper unwrapping - which is exactly what it is for. --- tests/e2e/.gitignore | 2 + tests/e2e/README.md | 103 +++ tests/e2e/__pycache__/run.cpython-314.pyc | Bin 0 -> 62994 bytes tests/e2e/catalog.yml | 375 +++++++++ tests/e2e/run.py | 931 ++++++++++++++++++++++ 5 files changed, 1411 insertions(+) create mode 100644 tests/e2e/.gitignore create mode 100644 tests/e2e/README.md create mode 100644 tests/e2e/__pycache__/run.cpython-314.pyc create mode 100644 tests/e2e/catalog.yml create mode 100755 tests/e2e/run.py diff --git a/tests/e2e/.gitignore b/tests/e2e/.gitignore new file mode 100644 index 0000000..aca4e0f --- /dev/null +++ b/tests/e2e/.gitignore @@ -0,0 +1,2 @@ +# Harness run output: trees, logs, results +runs/ diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 0000000..110872c --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,103 @@ +# pkh end-to-end package harness + +Exercises pkh the way it is used in anger: real packages, the whole +workflow. For every package in `catalog.yml` it runs + +``` +pkh pull --archive # sources from the Debian/Ubuntu archive +pkh chlog --rebuild # no-change rebuild entry (1.0-1 -> 1.0-1build1) +pkh build # .dsc + _source.changes next to the tree +pkh deb # binary packages in an ephemeral chroot +``` + +The catalog spans ~60 source packages: smoke-test smalls (`hello`, +`micro`, `jq`...), core utilities, server software, databases, desktop +stacks, toolchains and the weird corners — `dracut`, `opensbi`, `grub2`, +a cross `u-boot`, the kbuild-based `kbuild`, and (opt-in, hours each) +`gcc-14`, `glibc`, `rustc`, `libreoffice`, `firefox`, the Ubuntu `linux` +kernel... + +## Prerequisites + +- A pkh binary: the harness prefers the repo build + (`target/release/pkh`, then `target/debug/pkh`), falls back to `pkh` + on PATH; override with `--pkh PATH` or build first with `--build-pkh`. +- The pkh runtime toolchain: `mmdebstrap`, `sudo` (passwordless, for + the unshare chroot builds), `dpkg-dev`, `distro-info` data, and + network access to the Debian and Ubuntu archives. +- Python 3 only (stdlib; PyYAML is used when installed, with a built-in + fallback parser for the catalog's YAML subset). +- Disk: tens of GB under `runs/` for trees and artifacts, plus pkh's own + caches (`/var/tmp/pkh`); expect a full default-tier run to take many + hours — the tiers exist so you never have to. + +## Usage + +``` +./run.py --list # what would run, with tiers and notes +./run.py --dry-run # the exact commands, nothing executed +./run.py # smoke + standard tiers +./run.py --tier long # the multi-hour builds +./run.py --tier all # everything +./run.py --only hello,dracut # exact names +./run.py --category database # or server, desktop, boot-firmware... +./run.py --tag cross --arch arm64 # tag/arch sweeps +./run.py --stages pull # catalog sanity: can we pull everything? +./run.py --resume runs//results.json # continue an interrupted run +``` + +Useful global pass-throughs: `--dist`, `--series`, `--arch`, `--cross`, +`--pocket`, `--ppa user/ppa` (deb dependencies), `--jobs N`, +`--no-ccache`, `--git` (pull through the VCS instead of the archive). + +Exit codes: `0` nothing unexpected failed, `1` at least one package +failed (or xpassed under `--strict`), `2` usage/catalog error, `130` +interrupted. + +## Selection: tiers, categories, tags + +Entries carry a **tier**: `smoke` (minutes), `standard` (the default, +up to ~1-2h each), `long` (hours: compilers, browsers, the kernel — +only run when asked). Default selection is smoke + standard. On top of +that, filters compose: `--category`, `--tag`, `--only`, `--skip`. + +An entry can be marked `xfail: ` when it is known-broken; it +then reports `xfail` on failure and `xpass` on unexpected success +(xpass fails the run only under `--strict`). + +## What a run produces + +``` +runs// + logs/.log # full output of every stage of that package + // # the pulled source tree + /*.dsc ... # build artifacts, next to the tree as pkh puts them + results.json # machine-readable, written progressively + results.md # the summary table +``` + +The run directory is per invocation; `--resume /results.json` +reuses it, keeps the recorded outcomes and only reruns packages that +did not finish. Already-pulled trees are kept (`pkh pull` refuses +existing directories anyway — delete `runs///` to force a +re-pull), and stages that already succeeded are reused rather than +re-run, so a resumed `chlog` does not bump the version twice; change +`--dist`/`--series`/`--arch`/`--cross` and everything re-runs. + +Pulls run a few at a time (`--pull-jobs`), the build pipeline is +sequential by design: one heavy build at a time keeps pkh's shared +chroot tarball, apt and ccache caches warm, so repeat runs get +dramatically faster. + +Each stage gets the package's timeout budget (tier default, overridable +per entry with `timeout:`); on timeout the whole build process group is +killed and the package is marked `timeout`. + +## Adding packages + +Edit `catalog.yml`. Fields are documented at the top of the file; +keep **source** names as they exist in the archive — not binary names +(`make-dfsg` and `rust-ripgrep`, not `make` and `ripgrep`; +`xz-utils`, not `xz`; `postgresql-17`, not `postgresql`). The file +stays within a small YAML subset so the stdlib fallback parser can read +it: flat mappings, inline lists, no colons in values. diff --git a/tests/e2e/__pycache__/run.cpython-314.pyc b/tests/e2e/__pycache__/run.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ef52a8ff1754b2a285a5ea6d55110220bdf202dd GIT binary patch literal 62994 zcmeFa3v^q_c_w)ABwi#y0DQl|_xmA=5-Ey$P<%)fC5nP5N@`2uQUoN?qDb@ws0aID z+U+x@+MNh;w@oOvBPeZ;pvjrj?2VIVXJ%uyW4qC`IS+S|v&(v9d*mK@0mbD#3Xg)_l^zwlt365VuJLHtUF*@ZyUwFy zcfCi??go#6-IG1Z?4IIDVRxg)$nGYOiQQ8@skke8%zG>z%O0!8%D$`YN!yd|NiX2+ z)t(G{jVIHtI+<@*pS)yGvTN!Uo-BKbioX7SVO4}9BC&oNQXBx%>+#&s1ZdMG z*2+_APsa0<3~tQiDYF|9H?erRJr!{?i&xk!h+A2_(w>HRI*V7?GZ4>Y@oIY(;@K=- zW6wc6m&I%Cd5Gt;c%8k$UVvXV`ZXr=)U$dO;z^NMFHeKL7{5zc`VIC{#LHN`(O!;t z1&ePy(PXb|Im&U48jj25P8j;WJL#*2}xYboR+{w~coWkBRTxw$uE?dE3aa zXLxMlOx@-4V>ZuOeq!>>S=^nr)01OJic&95@aImCO1?>@k&E8hDp-C>JIs%q z9lhYJv7NeXb2v{APmXyiV~I!3qIg?vE$=)vIXdP*5f$STwIgST$5B7=p>y2BU*2l# z>g*XfV7HCBZEnxxsZ&meZDKN^D6s&lhj3NYIou<*YTJ-|Vv-+m)(OSBZR5^M9-C)^ zwZy|ao$=~8oTtRhVi~7K$A|gLakz|*+lI$&PS;uIc_$BmjQ}hYsD`fVf)nZ79)c|D zh<2Sp6F{7*!l|jsdDp4;52; zYGQKSQHiRcX9S43MqSRa(Q$yAkihMHb`tO&MYhYfiU~YE>m0stStq0&_SlA5D}+X% zyUURUc!ZV+fT$mKdFn?-hS5G?4?^m#;mfv*!~FS5VA^3f;M2Nm6C?Q6 zxM$ciIx$|StCU6b!{g%J{#4ZGbD)#*{^m;-5){llKK5fuYnL?c2j zq9hQFp2c)9l1O!;(}U~*QTu7`1m}@q!jqqD_9*N!kJ2vpsO*Yju2LB>bOO^Aa*D^lfT>wsvyu7@b>&h>D`bhgu>lpTieVL=-O9@c)hvunS{gY+~3GNg`^D86={* zbb5FcC1T9bS3tnsBLvzgcsb%Y6x8{-h)i>X+(~)MmoazaSuS@_c2;JW+2!>zfJpHR zN(&rwdM5dCz8pUo>Y;jSjM?+U9yb+Q#9JsCa5_3r3B%0{xOIuo_9K-$9lq!cUZVHTxB&y5_TyOqE^}mFCs!U?H8V}3aM6fIg+XV(rJph<550~Y zpMnAk3MnX}fZz_&34G&A={Gf*ucV-gf@%tg&hV)eP!)vA5E}4lVz3nv_tz18h5Na3 z*GIbKmwJ}1+0%PwMncN$WmEQa=S(q+WeTy(WwUj^z5a+Wpl=K*GyAL znGt{aL0Wd8X)&!~x??6Qq-I{7%8+uy zC(1b12{oq5{pAC zkPXWElh%G8^M_n4)5IN?Ps;1KNqP4cj(fIfz0$efovSkm)(er*9JZmnoPem9wFt&5K<21P06|XauP>ePeuy&1dl}QoV=OVHb z5n0WYyu1jw$xk|e@z1y_<$OCbW3Ct;b32hvg|R*Ais-;y@E-TYQ7}&$$WlT~<#vsY zdLoK3=XfM()a@8OL%+b*RVw&4R?O)5IA%841vlYbnaw7UxuhZY$ne-O&l5po*d9Ry zL``qJ+Ipq+t8LSr%SrmLeBp~<@VdiEIqxOq_?5SkZzcyu!nVfuY>mq%tM{On_v*qX z+k(j!G8N6JmK!#`rGHaD-?7-RGi2;rXxJIp@b2z+cE8)RXe#r{W|OWZy{erxUNibt z@0-f-dcoKi)b!m;(#Gn&0kU%edEWTGw8y`TQl z$e9kv+(dQXDb46iQT{+yjBphxAu7rSu#EvhdY-i_?8pEkG<6X%^`uOkC{0|KtowiAxQn1!eAl2#YO)n`#qb*}8b*ssih77?`(q;(wT@nP zr8vH0C7~s`TyF=*9gmHxL6w*)@tM@O?6H($jvA&e6F%osuiqM1(xCbz$0$QFYfqBh z$i_U%KW3-J@ z*=6jzcHv9?H2JT&>^iYc7Xc&wic2@B6`^)U`WyPhZycwYo?U~nk_N#jCQ`*g&3e6! zP6lrDqMX4b$=)=GH5BXRfbuSvtLIz_d})JaA_WBP}4yjwUt-Or?g^?Tc@;Qm0S5; zK$c2fMC%-fYJrE0hKY}+I1;$sd?(&U)MIQ$^I?n1W5HlF-(S!DEQ%v5%uV}0|F)KBQV*zoQ{Zs8mw?TT@e*)p_}Lw z=1=hw=26oSzUp?8v@kk!dHDPoe;UacmHz=q@-+8RYL3tD&kopwCa}RYC3P-m=1zCs z%dq*aw+e0+gbEwy+QS)}ujpsg%SQ9n%U3R6eeUJwd?PnLfBo~{91G`fTFBotHxkZo zeJ{WD1B3Nms?~ew)y!{o_!YNwH+6xHuNy*hmk>gcuD>uJ?SichOMskuf7|ELt!Y`J2MV ziua5a%NbdypUL9g`*mX&EP|=b?^!g}Eaw#acYV7poKw4yQya>u_i8@OD2P~dG5^2( ze8Lw8gT+Te=|_X6qo1jEw(@}L^^}>;*{o|>ujVWo^6qcrGMiVna|Yw|-uP@R&aj|7 zpP5;RC|yu`@x;jU#4SA3GZg_7FHby?X?O)WVobwkndLr*!0TC|n_tg<#jA<5>>_yy z6Nv1=OcLQC*aEQu(FAm}gP9POIWAd(37>V<3<(w&XDoNJ)S| zO2T(j#C)qDkYX=rjtUnpNQ6iHe!N7CZ$W=L(EpE&OcfiMHtGCq%*JRWL=a;? zaiD81d+`%WFO<$FQ95ZL*IXh^a-b~`u#Ekrn_P2=Y0gN|9-)Ort7wm=zjjR#_&I14 z;amVp+dCg+vNgq=jXO^2R(7eQ8uSA0EO#9E7<&qQ%;j9Ff%xnNx}FGcyK;c&ziL2b z;NrCzP@R>Zg*AjrwL>*38|Aht^0-l1RN0d}Dyn@_p!O-5QcZeJ*S0|Q3?A;FM)~2yw15FxLJffaFH978?tjvgLVQAnOA~kSCKH-ie zIma)I@{kucQ8Fd5?-8wg__T8f%QOzg$+LyJh_>s}h?A`ql0@L)Nns|aemW_qV%Xsj zl<78lL8#0i7*WyM!yy5NMC%0Y{pS#?H1TKY0n^dxCLuw%E;I3gr>DmeL=?_TBdpIO zdPpckkWScI6e=x5uyg^XpKO%>pC}7;enjSq$grwJDK5JSu@mGKCbtMei-DRq6qO43 zFCrb|FzdiB3L3!eSGN0`XSRp*)euzDGG?E-_RQ?bYbS#x&7riG#nhJRy(@BqD&<3c z+TR&6J~F4z)?KUfTSMluD;@t}1~(rvS6t~>F>xl#)r(gydZ$9hA~5(tV}<|J^qysN z+HCo?a$i-*TpBi42hG(1^K|cr`t*A#**=F4Onh@NrzMoKC8*i*QBL0U?t5vOuO3E4 z%sIZ^FiKX>$Vkbzaklwdvv0G%F=QzV8Oy`Q+67~6$XGw4{LpA2*`>sv?tgA>19XU+ z=E~=Dg9Y6o^X{NwH$<3Mm~evx!Sc5Gf%)z`g~5W(khv>p=vvkou4=Amyw+Dz{=t;C zoSKHu`z$vyu4h1s8w_PNh0-=L8Ep2*wIjYk|Iu5=Zyuj3o39F$c7@WqL#ex^-d2Q4 zJ3?ulq13K>mUORs_TsgRzDogfD5EB1sZGpa52aOvQY-J7#Z(sqs!&E<$XqXxak;4U zR{72HKJLBg=>FADXiAT%DWw)AdHU~Dpz9U@R5-e_+JB1*W+PYBO;yd_S zv3K|zWr!}Mwtk$GlB%0gt>kltRMH`T>+qt!NbCnbP!dQF*6+BZdRO<3ZvOeZ8-fM< zL*~Aqp>J7lqU6E64U75>Pnmw})vYsIeUmq)ZcO>6*5uh6Ea(lH_XQ36mNmw!DOXaw zlVMHX{jHp(a;1$kSf=}a{qZpwXUO~*Ynmy`n&f+W>&Kjep)(bg7w`m5&+VFX&pCqy zZ6WivI~#+BZlwFoN)l52-@k0-Qn!5iOATk<>L$W5pVTMg-r>?HOxGj4yUE(un*CRs zt$jO+{zhi)YgYVCx{Sh%oW4fI-!!TaU!{IlqZB5p(OgfWn$$8csh{mCySkpN9g+kU z^m9nhW8DYDYnVUderQI>CXBj1vJ8P%#mU6YsXPy8zVmYlLT3$|HP|BclLgSf!YAB9 zGWc}>%>L98a1=YfY z3up+}aitg&C-(i#()lIDfS8z;u%ZP911nk*T53^@0*hviF{yGAT1{N{MzFNXL1nx? z#4ttIyin)3#@eo6I%4!ye@t&IwgKrLX%E;5b)qJjnBzfWrd3f^5|sVf!%`3%WGh(F zI;4U5QK9CTaH#~B+ND}m3Z|dOmJ)9XE2TfiD8=iIl8d7yC!u6nKia(Z5?MIMEym_a zrg>s7X(r_jV3@kWB0f8_UN4DXOIUZ*te5_rycd!$gN@duId&ld!UAk0S~RFl=;fz? z4c07GH3_{=D-8+X5qeVkI{@Z7Jt^x4)~>xo2sqx2!doKE6XA_@-Wa^s9Tno&65uU= zYIw_6!FvPx_E^}2^wPMB(Ha#p*Xs`y_`B323JF#Z zjE~BMktgdXHhRq!t3OP+AO=aMd18O*68h^utOtMbYYF|OTQ7aW_|Us_$BA=h*ReSP zgBg|p+Mps5Fh_3SbqwZMNfR%z_go3eDF;khK;>gAe=rZ1q?U1262z?-Ek7$;g-ycf zm8jF8{$FAAN`D89U3c`7@U!L;`zJ;%CDQy`!G@=XH`PpEv4bTX2x1Q z?;Sa_5zF*(TZ?E+6%)kcF^$Bt_*?n9Z*}XwjhFJ6bd{Q^J?$p{R zalx(?dW0HVL{>SKTU$$}#iT~?2v)u^yUJEurBRUoTTvc=k%HY6bW%VAJEHvj#OQcj zzKXBf7fD0S7dnp255JZ4 zX40Q&5nZT1x}xAzx(8}5HGM`2Ir6p1_w~6XG~~}I0~h8_2u1w#aH!)@aMwU^;Bc^D zFl0UwG#t6FLZ15?F1I*j+{(Bz;g&9tU)%gZGdw|Z{& z+&Xac0BTaw5;Ao!l(hJsjn*W~n>w3uE#n(mcsQd9n!4|qGks}u<)OmXH_HPDZMf!=0iclp=d)o85q@X zs^^^x4IPP1=v>ltMw_sefpPQg&2xuuCqQxQZ(aB3$9Kj)Uizjn@!LC>G&`f;-XVT_ zJ`E6z18&E%-V)U329>#={&EY~a7^~;FAbcj`O`?s_D_GQ=FIzKpGGV__}lsE-=}ut z-jBcCpZ>iG@xoS0V(p?NTZKnR7P0Jb{|QFdUu5<-DSwb|?%$&PL9Lp?E$061%D*(3 z5dJVV&|Qdaz9Q(=zkHYV}XInGpUg+kCJ=`Lhjbgnv_! zad4CJZHmq3#nyH;vhDXd1iqRTx&=0}UyrWs#;W?gXjM<6;<;GM zg~24IM&XJZACk%G$+i}V2co$ngid)QCP|n(D%f+dt6)iyswP2jM=B+IMhc4PWmhFr zUZs$CO=>ljSCf!>b$KkWlJZt2!8oNhB6E+LFu_M;=Oc=SMv(Wjl`?)WzJ%L1aZ5DJ zHk>~18FEa*B!nzHUdLMox(P&}v`k9*(ic83n(w8gddt5$HVyKS;x&Ky`DJsKH_KNz z^FmOQOOLEyei4s+_OI7^3;Z&Fd0^LfYXa_I;l{ZQLF1;NX47&?n$P-s6T#$s->^R; z(DdE>xm`=P&2#Qx!PfZ=LH)L%a$CGVL|DiA!^Ud!cm+-Z>In1TC^CEoZ21()-DC$& z=7T0#JM8i)^yRFudjO^Z^gCWdBxM(|Q4G1g=wFx+0#~RXNvXu{P%N=K_Gz*^R-SR! zH|pwSE*U8jU@VAMx?pw)g^0XAT5RGi#?Z1gDy;P5duuPd0eJ~pYpz809!6nuDI5Q% zDnD5|@9OgHl7j28#{LAcMXX@LIv+TVT#nVwAij;?>sGC+$Algw#5@T&pqA7lYc8Qr z@J>QSAoLU7NeKBu-{Bq9R%@?lf2K%@e#o(oxXmy?PRguCz1SYCs>ynj1VPVklq@r5 z%V9Sir_~>J#c2^}N@}Ab(FOt3z#K{HZ|~@B-`(|4(bm|&4HF z#LY%jf_2(87N<=H!`N#?dwNuO!gy1GDb@2ZNOST-q=)6-#5*>g>p@0fD5i&9*G#=H z6-ydLzH0wL|AoLQ|GBx;xz_oSVB@Ykg=EuJJX7O6y`qq5^H&s#Wc75{#~LmrZAOM| z7%;%r8?H8AZJudfh6aA$m3=6w@K)K)GJj?;t7_3uP48Q-w#>AM@0sYYaN$PK#X$QTyKnCf+M1R!n?lA-V0tylR}EJT%SO}H3s)|9 zM`p%+g}(M1-PgPQjlQk^;ehN7^=);4$I46LT>D#l-`qRz4sPtYvq3Cec0+w#9Wufk zEtpc`Z}&U=gJ@53?I#L_R(Ics)~uv+2Gi9oSGIT$hxD*J|IVR6#_PxZeRH+J{GChs zok8W!IDr#MFb!Qs;0U$Wo4rYgPTSG#AaV67AfH~)rN4xH{Ff=1rhv3lWO|Q1qiiC_ zPOfta23&jzB_ag|rmA|}B)5^IqIPa*yHFw0o5R{0K`^swh3<2k*zJym-R?HATX5h6 zOC&TS3&zd=Nc8Ghdw;@=E*547O_!p>64Xv6 zm99>)O3eu_VLgg{hOMH55+-m4VohRHy^3l(fbERZWl7ddqP#l!BQ?{A--_SGuf}S& zP8x?;GX-iUkxsH^66qvrMo7Eniq#Cb1&&Npu40Dd&qPzHA4&CZ zw?~BfC$ajoITfwcJW_ulm6lR<$sAvdw)4?+|90ES^-J&xlGq}V4vQ=*5f_AQMSzlS zQ1u8F$)q?5F=rINx%JXGL`>5^(g%|GzNT!M)R+*{{Hu)}$;Bm%PYKS6CIs=qLLT@o zvI+$!s$L6AQH70iB)F_tn*{z=X-`g+wWM;0T+`a`pnM5VEmpqRA1ROY2T4vw*b>9h z0_^P__;cgWHKnbsjrzL4?qu?7K~JJSEFvnl3!0w5O~k1Qw=<%^HfQ7;ewb8S%h;ud zs@hs?RK~N>3kF^GbykMTcj4{8yfVHfSFCt)`#Rza9z@RU2E#m4w( z{v@7FtJ72n=?HidSR;y&^9~^^<#Hz>D8mH{6@g7kX4CG_$i(^caI3(-gZFGEZDq2{ zJ$=fI`>QS9!$DmREb5aCUiFv1u&mL~>aOX0o&J(rl{YKHMO%VJTLP1BOx>Oe8e5k% ztuqR!>#hoR1En`=Zq)c{mJLR4r?2Ej_4Vp-_NHL=ra;;oxwmtJmd#6s%`;@H|Jl+T zEmtpHy)<))%I)=TTQb;YI#wE(75Pd%r!#uX7qwZas8>Jp!jdNUv)5>)LMW_Us<4$3 z;bk$sEd9Ow-{nIvtfzelVPjd)Smx`xvG@93Sk>r_QHXilmJ9_1AX5}TB{!;WRQamt zb&q%Jk|F=MfSx6j_AdB5H!fYj6tc-xw>U$ z3#-ewihYh7qt{2jSr@caE*UC??{2@keP%no-|yYAWGJGCDN>gF#27J_ictG;JfpC^ zUf>bykd8kJm)k$jt|H{y~0r;8kbL0a1oo zAvztSaC!i*EO4=d>4|O~f)%9#Hhhftj?xY- zs~pt+7x-N63#jv37t_b)4xzS8%_K5uKVpkYzD zLHxPRU%9BPc?{wZ-^LJ!=9b4HPSjvWDT*6R*w6`xbO-sE2mxAn(ZTkGL^Pn=*iD3O zBF*%crUPNGNUC7G1Cg-?d%)m8;>iX(+X?-IW`DvJ$p)Y&qHtMjUQw_}R$eqxSsRVC zwF{AtRO$Csl;%+pgxC~@re8*gX3HhFl*AMOTV+Eqf}KN#eGi0&YOvUm-q^ z-=jNV(UVXd>-Wp-hNp$YfGTmf?MC$Qq#V*T{)vv$3b5DRmS9@lFR``_BkXSaHsDXn z7j_M(kK3YAPRbZQPM++V-@3N)JWjgCaZ`KLZOgiKBTBpGI;pekngLC^e!DiE)a?jeK~YkqmCwTjXJZkorWnkEmLD{cmg|F&pGSC)VW5-V9hN~ zB7#3JDuU`vE5~T?C{bf&o8PxIGSX=R4#Q7_eBxY0NwBU=!&*5ut4358dF)L^xo5|mmw4g^ zB5Jlx0MAa3O}fv*Q3QMkMYQK9pkJIAA05GTIC(1CBN0_UmBJ>zNK$9lU>|1Fi2Be# zCtTm~OH^9QfZ%$l>yisTy&Mt!5J7&30S@&aJu^N$7D)ocdV7#Hwbyz1)CByR^@x7L z*gJw|1kU2fN&#CK;G}}TgTe*C{FGuegTX1q#2KbS<2{gBfHaSC`dV}(xy^!-?f*a) zHf`+%Mq`2A%IVBk^RDE1Jz;I`qBhsB07rXOdqwN*{8rkH>_5o%cYL!Tm{}RfoGYG7 zn;Qw5Tf^q|1#^4I+_9+Xj3$0Xf6ru{P5D|%$Yh&Q-7{u|jRo%+3&0#=o9R7M)~ot^ z=G?HkWWij5-Lv;Bd0|V*f~90RzvyG5#-y83u9$UN^D^|2Jy&{s=K{Ny^6O@L77g{N zUsm1?<8@iM!V{13)M`E_$?OZl52fGij`e*#9Ha(`UH znF>GVR9f@LYA!ib%sAHwA!SWLhSU{3(mnj76j=eapf>lRo4Vo;b`^HlseaIw)oD`x zWm*H`KQy%`cULFx0(p}?ARK-I&%+d~ z^(3uo1%Brrk!Q*j$J9E8sMt&~dS)s`z`D9a4Efv)Px)^l1LJ-oO17Z-Bz~K!>aJlm z`-IcUlJ8Tntjk_e(7mu;xHosO`$sD6eHESy98Q#GBQH2`{IN#<5|L>xhdWMB*(Ie$ zElhl`6jR3U#1TmGGa_$-4Xz}E;3~kkkd~Uc<|1`&d@iRYUd*i28SKJLh40cVn#b)c ziq9YG&yL82mI}G3L@^(B%q*hVzDf;G{B}f5#KCnYk_3|KIx~t*@h$}BeqjRGH!*Q; z(#72E37z&K^6>u^0i4fJkMsW*1w?lQ=QCx3k1hTm=rIMX*G?gV@vc^=lJ9ABSG6x| zUn`oe`dXE5;KtGGN3p3Spjgz@u=lh#0~EklGNXN8Q}FALlQ=^@S@9=-pfO+(L6!U) zEQc7|&70fvxOWWt_H4yFSuzUq(%LsG-q|EWID%BM zE7~`*2UUzDiM|KzGWZ^p!%v-p`Kg177I77egI7a5iN)1+4dQSq3tx6gb{*nyBa67k zZa^HaV-eSq`&e=vi@1*5$CB$<#P#GpmR!doZXoxuyw zxsQ$d%`=hvSaKbU=c(jAmK?_-ZYJlkcDRg1++r_69PVNfx01_Pau}4 zlgn6pC5va+s}Qed@l1OS;erzU5o@$Et32FYAc8?W7A?)_R2`6zC=BM+ zudv8`wQ#@NDZF@88j8{i?Sml0jw7aVlqOt9Y~y$qjW>)(@rWavUuyrFOG=7~djo^@ ztune3pbFNvol-PHlr&;IBOna#I@ou&zU%|LNzlxw)h-rs#*t6B>OqahV|8V}qB01V z$k$7#N#pVoN+3a*U7&5WlDg&+>y4BK@fA`^*iOTLO#*F1`q+}lW75kL)Az@=p^IMu z)2tu?y*b)9;Ht3}7_}5tVxy*---)zS2GPMSdAubbrDxm&@N1fcT{ci}xUq@z64V_- z6X*J1HOWwvVdwXdm)JotHtx9fWF!f0R3~_cJ7ORe?ohPsh!!$GHYS24jVSlQt0a8~ zVe$}hb3?=;fg48VXhf_NQx-&2=g;9SR-!lKn`^4T-@r3r5t{U?f_4>dPNAdm!11Fg zJ}`tQY*74r&IMbSCojLLnapLt zsTZNdTt-K+@*RT};VMEy9Sd}R8sC`$kHVCDG^p2A?$L_lLlGTYdECXjnk)#8M7-JgIK!g=y#=VCQ^mT#&JDdW~N|O^DKe>szvMu2JKhqPr z18$(;x7x`hZ3Xt9Gp|sm*-zb}4tr1kz))w8J<1{zYdq?}d_nXVUI5{1ivfeb%!(H% zJ`;n_yWBLR3byKu7V}C#Es_=^!-@2s5t(ef7q(=>944&K!T+E>XGN||J|er9kuy8> zwW;rv_&v8S-@F_yZ(b;G4i&Y8i*_v(?FtojhBCUm%6r-QK99fW+h4$@UvJV1%y)BU zw_n>HwiLf-DTeQ)U}@K#Gk1;!iw}g-`-7(bM(V)aL#u-p+ z-1JpZ3i~PIaTY+-6tV`8x)x}MRX{j4_MKibmeBU?fXDyBlCdeMX^KyB_yD^;6M%0= zJ!noNtKc``=>hzWL;x#QmG?OWKIaPE0!sGlQ2_m~IVra?%9%p?tP$>okTVOAd+I=s-{GKUKFyjcM&8g>=-`BjY37OjAVwyM) zx+Pyr_AwF}GF2{{(!!>~1yfx zfv9veXW3f{HngrZ3@vfUBfw|Hzb}0qQX` zxA^QTK4GBVkK(4ed&bOrhAdz}<@HMc;lS>>fwzvnc{F5zWe{7<@j63>T#E0%vfo#F zqw0E9$WRK)_^=^w!H^d+6ac$Sx)rRv=zd2mk)BPaiUqrlN31)Z+BFO5BL0GKi-c90 zZ8WN5&|^|a*hTkXMX&H+9sEj15s5R-|CjL#^BmWo3UI8s;<%0NOPrKf>LX^%0J7K! zQ84N(=jTz-lpd#f)kn|pf~@o)(rlCy~II6Y-inH=oAw7G-aNq(`1L;$t@s-sA*YYYsG@KB9Ioc*3W3c zX@!j?3&s-v;klmq7edDUcP|HxM}wN9zvJlOY1Q|MH1zIjJm16ce2>i6MR>kP7A)%_ zJl_+wT1w=YVs!NX$8(?v_ZU2X8?PWY!ZP?XAb2v8$B~(DM;|?k(Mlw8uR&%tkQt97 zDGAI3ezQ)NMC7iyrng$&YzZ0KJ_GfRBC^REHWn@z3;m@bV+D3FB_ML=JY2`M-)RaO zcLz1Qe@8@?Ci73;DMV!aJwk8NND>)W5sQGp51|3Kz-9L9Q3_53f^i6qhzxVXvP3FA z2$Rr_SUbkroW?FQ#jM!G`I@u^OhZt*+!IS#wJcN zHnvg~k=HfQLJXK!oOtyjD$P$*Km=9XcsD&Q5KnE}87Dcu=eI|yM3h@Yoa@@K=z=3x zw%5hdhNx4vp}1fF0-%73Azk_tToavK1AV zoN~4M_q%_d)(nRa$(Fy%Y59~X{a$M4+7&W*y;oJy(U4Q7sx?PLf-U-es(p;vIel7C z5$;4KA{mdVG(#I&-1?OGowQ#?tN1{Lsy^V9#J(thXZD5PhblagKz~5M61OJHRC__7 zDfDBP8{GrPgx`;1G4ZJki%%hvh{1%rn5*$_?G?u?sgoAsXCplY*3(SEupVg6#c&?d z>9eU)>oieZg?vi5x(Y{c%LK%f^E6Z#LJEouVFKsv)R7fpN3Nor61W#3a9U9(LcL&G z&qCQAf9mY%Z=4R5?fLY95-+}#nJr`J1d^=~kcX*J4$#+QXe4U;cGK5mn&wqsXFW#? zGYp3o(t7-kTjy_{pX&;hwT8=fER^lIleAd2CzRF`GWG;DJ&%$}(xjuCWDjasDUaX( zp0N2+nP?9^jeLS8keDF;XY_Ly#Yzx^wcOd^=)qk~QAn~0#!EeJ%=BA0)sAnVTtrOo zA*Y)p4`FRTomTMEm($1?2=|gr!hKh-a6g3or+EI6HXBAks&t`sB>jq4bOA3pY6dbW zUw!0M)NvtCYJ)*VY=P_Z?Du1IWKxO|kuXc5v|M~EVN{KsV$wGMf+NC4g_N)t6U+cB zx@ZBM{>4prae$2+Dy7&m8O8~DjKJUe=BfMg1pmj_J+QqUMxyXiD_Euqr+>f^ag4ny14jL!YGDkAg^ceeZ(00T{%4Iy@KcL|6=#{W;Bg`BO_ZZrpAl&=!crUEj zu-!=lIVK(_k|~bjwfsE8yo<=kl)DDAxA6D2_>_zK+*`-sNp?2xS{^wqy*?F6tqP~s zFQnGbDZ{Co7E?FPTVRhktG%XOPTzCadRKk-RM2`bY(2VQJsPq;^RYsnYW_sQnKwga zoR;J3_HX!hPtZ~pG?XnH(`J%C(c={gdMF%or7hVdSN%Y)-(_BH!ly*V*f6!R8a{rQ zCa!`^BW;3`KAnC(X#oB=04c`J=&_-IL*SoWq1zmMRN(gLJP_m4zsG9#WZSzs>;X7H z8<`UaEArA6pCO>9WXEJSM^iB_L_#;R-XI~AKTbrBU8}6;;EgqAJ*RqTak$Q%sbwx$ z1#uB5Jph*YuaPm5GJ+F|f#^f96?RQ}qUW=K+-T zMEv|D5kJcaKN0wVpST6A?AN3CN!l%pp5zB9z|_aB(n)hGQe3Y{&|Pwz=0+znQtQ`T zr=VlU8wqU18|;FS(%j<>ZO4ycYrIG)?cLLEM@FgeCo5TMO8g6AOtMsobSM=rgCt9p zSRY6zwTdQJ?Cg=;2Aq%%)QaseA>?D1)W+Mm%?0X{gffZ7t+^7)%!qzly516bQK|-| zX09m`Kk-J3Hwja$mrk%q0vqB$6LAgw-;$*~*QDiB6G_C;y2FUm?OSZQ28@waQPM;WpL?nr}_ z>>v$yD%qY!|KhDizY%l0<`P*puxjZiZFcR+Lc8^3VZEfIrM0a(Ss+@P!#-()Sg<`q zx=yey$Yj<^S!>I|o9yF|mxK#%Qxjn)Vggd-G-@B?0xDNb73+s&2J1YOmYY}_*@FT8 zDTzQA^J45{=6wPq`W=B0#oYarSUIo?iw2@t1WPoD5gMyK!y=@80v1sUss=2t5*rkS zq!bn%lf)v_Mq>0zVbOle`taGW5a5Fv{f^kSw)Kz0HtD((w$Xtq_N=w#tj0FHT?g9+ zB*%gMQEUSp#>mb?X^&uAV#mZWZ4}2|J?>3Z)XAT!>c--z%{A(Bk}oK>RhqWsv72R9kz~&V4tm6%Gq$sjC!AnH49=N%4ZGcabPAo@&2Pu^Ddh=U&gXbPMb$Z< zYdCMK;IH5Z4iBN_=SgCgZGS)gnyoUMS;YzVahUp|F0g1o2*$=VMa9pC!ReDYrIJpZ zk0hNR#vVU7LzHNcU{>{0WX0hoPUIN9;2gp>auu4v{}(Fpf2DUiX8g=x$Sdg8J$kBg zVfUVgxd2QQ-|%VPd0}({3)pNWC5i3A!yY%)$ix^{)YZt#7POSr`7kjiw+kn*}__g>q(qQI|@4Cy}A zjg;#tD=Pd#)?`ccZ4&*~(C<#-rDyxD^@XhsL2H9xwz8t7_c|^)&Fh#Qy*3(7FIz}2 z3#V@mrf&|EzEN|#W<^ga3~Ng04qCf~62exBe{WB?sq?+2&QMeLN-||g;gYT1!?T~e z_PKD{#$eh;f2Zg!b;U^UP4OzG(rg#lav#cf=&`L&H+&+H4=Uhp5j^~}v@LKzL=jJAagoWk<` z<8L1iWdO3hD}_irqsGoRO9n|GA#>48(uzDqn?lO}16L0Cx&tkXhNgcgtGYG*N8_Qg z&GQ*cWjklOy*m~R#mh$P)#t80H}f1!7w1No>UZAhUaa4Hb9b=+h*#~s49fu$#j|F! zUd>vr-oT6-58rs^`ZEir3KFq%KEgNlUfKJ~zGX<-b~u473Y?kagVxrtb^C&K`}~DF zqf0ybgZ6=t_3%vBJwxiUIoikGHs3}6r2kCdAbbJSmxFcf(0=ZcS?^hG{`7!xsjwwv z-4e94&KE)|O5OHR!)9{DJKr2`=y;Bz_16l0uSNUU>FqiFO2tn%WbLK)auH+->%}+=e3dWQ6-w|VwMhY#(QeXklHPK^OjUfJg?{S_y9cnQ zPfzEc6Yh7hjKpz6_(mwcaNmv+?4J)R(Lkc?p7#CxNBb%{N9Z7|2tL&g+ zFXymD#T$s1SU!@m<)iYrBE8$d#kQtL4`RUApbD4oPh%=^#r{+nk3A;0IH|gnhhmyi z2Nr&EQ9mis87HpC%>DP{*mD;&lwgh%+ZwI?7Tg_wL?z)pTD|syP=TJI3)=z+KGADE zwNH%2t7R-h)M+5ZFGaoL5X9_6X4S5u={Dg=1M+kVSgpOl(j>=^r7qz1_n^KKqOlY- zq)W6{h^##`WIF>9Rt>Q-sa`H!N4(XvKu0O~{j#SFs)^A?S|ej3$>c32e6#aPe}bhu z&5>zIr~{@^EC3rksEk1j0yla>CD{s%*b`vxIhQtmyt(5)v0PfIT=5MSsQpoV^}Cc7 z8`HEM5P#7lvJTnW_6};OMS~i9D$BtH_s8Iw*=`Y`h*HdyZ~qbGw6dJ|2@I9l)3(aD zt~%ZveFyAad&O}r(Q_X;onYlwqug{>Zs|JZN*O!Ga9FC`r^3CAK@G9n@dJmj@cIBS z`WGKFX9S@xi+wM3dEsYyU3ag;EhyFX%hEMVh> zD9mWxcg+=V3BgQ&n@hJ2MX(Y^qBf6WwoVGiVzfhmae)+!p9*%i$HOk6-SXac)^0Xh z&VcIY(y5-k3|~oBT60OZUcXN3(;wS<{Zq8Q@V9P#(PLU)oOmLx2nS$PNn>n#89UeX zr_tr-QuJIIwSmS|Wn~{@ZA74`5P_yf_Tck~B-nFQFnbP3-%asV6OhU1bSV}`_xSsk zJscNZ3x|LzwHCe!>9O*`*1|^+=ZQ07l9^x&WQ*En2CDUPqA-OK(LIcneRh~ASV}{u-R;6_-vavXR9C^YuoS`?P|M>)kdcq!eZr=ioNHb zMOm~j4X1fJn2jrco=T@-%FGCb745h2lcaB(4eqF^Od&(%lm&~H?uki$1U_!)GZ3O} zQ`@$}H_gb{q=PxW5h}vA6_a-!VGA7dd;^Cz@cA=5wt7b$ykV0!NZV8jeNTw9Pm-wr zP-&~(ZktjwJD6w^Ax=rA$ZQK$q>L0}&lHt4wkfmi6u!ix{lg>Yu)!I=w0RdbbPNG{ ze6>P=N&nd45hf@Qcq>!+e@Ah`51}{MvX+HPxIh6@GH{4RjQu_RUHf|ax`qzH(RbG& zo@`-wvSi@5QgD`n*QuNiiuF(cyyPkk!lD@C&!_1<;a)^TkRCoY=8Wh`Qy}=AaSJHG zdjC(5lxY_xc%iTMARAj!EjwLy8h~JzuntU)4I<`8v6k`h{~MTZSmZ$5x90kr;L7k5 zPGN;D4uUs2{NDFh^S1=-i|Gw_ zie~g5!3+2RiSvtk8`;KV&Dd8OGTHnsOQ!0uapQtw=QA8I$vIp%xrxR-01 z<{XeXYuX?IUs^)qbV$26-qa@(ET4V3!P4Eq^gT=4%H&ux5Xv z=F&^OLo;2F%Ww2v?_D&Oe_+b|ux_)rclN-w1OBc>Yt?+`OxK4-t1su)&`P#|E6x$Z6x6M04)tyV$ z&X5(G5)z=}9|)8NGUg1y^lgi#?Wlj^im3n6lC?f;X#)vuyb&y^@kc^xx2e zAgu-TDK*1;@tP5*hNfnGgq9{P!U<7P*{#}}wR5Ua(Pjo)bk~LX%XiCzHFmMB84K9) zAi}mTa3OFuSk^4SHe$`j**D>m)`gPR`AwmccH|-L1}eUso&hbC?z+zVynp-mYUVG7 z8g~ab?71roTK6t*+B)A8-qgLgsoQ7uo%SDF%C8URH7sOoxV!6D3Rzme44&9i&A?rA zhOc}vwQ#w9;~OvBeqsJ_sJ>&tRD&;g)BRhPA6Hpyt#~eMD)wJqGBsd}#^Wo$8o(#N zx9?6ysHrEoaqr#2VEVqWY5#I+y03ILKk9m5&Kas|L(dD}Z(6S2I5#+d>~2!9s$T>a zLk#iz9t?-X@7Dwd1AF`vG1#pEQ``3p^G8ArUBUY9J0tfpvVBdjUiO}Y2Yd{hv`q{o z_lIOa`B0Li{Fj7`wg&fRW(hd;B_05D_xTF@SV-g1A%oA#3Ncd7r zu9sSjW?TLw7P`dzQj5{{^h8|>x%WJ2W#I8nxk-jetRAQE#g0Eon4 zXX1ZQTE)S3V#gTSy}2hH6Sm3m;R~>$Bj#qS&FMPp#8MHM!I87H2N|Yu@Ym~x`5m}e zGPaw_lqi%uFWMd|>KaZ@>2}*r69)>1j#IjL(uZjYEFT<5GVeC6I8U&ee+gBsR77;= z&x2QlquOB?9nj}K3oki1_YXCMozkn+9C__H-;U;u;%_HdK{9UQPvecSBJ@4_$+*C{ znG;*z+K43Zj(BzyO?sIU{Q^Nm=NNUL8v;{@&5^Wp1aFF{H{$`BF9~s*Bk|$bNIWqC z--c1w6A?8f9Xcagii_K9v|tTke0b)pC%!5ad&DU4NfZW4S;08!Ub4w+oz1+K`PCD? z-k_mk-g>7K)_NA*OcLDnX;ZvLzw(ymrp7lFFb8(O(RaHqT(fhbW@o5oSGcApSkrT7 zBv?%|Udp~j&3<^)#_CY-wcbU2!F@HCw;hJu87V){-RAAQXUUpv`&ye{2?MK7aF$@2 zSNTEiHZnMeb{L61!2&@B=Coku<~h$>m*2dM*|fPQxOs0dy?4>H51tRNzWDNs@8vh$ z8CW##4I2;Rf6#asIacK0P(G$PAI{Uvnc>u;_fm`eO}E-^wuMp~U{$yP^F-BBDvlAP z6KHn5vK4C{K`Z9dxsJfTc^%F?YJ^aM57Dr%7H;$^4cD+dz+BKOV}e3T9RZ zt9Od0zbJ3$uj@m`l3+?Hc&ZXw*9zb$8(H#}f1>kWKK*3@+QL}wAIP)2iOCwcE});iSRF{IIi>9}L^3oXKO~P4ddRO(O3I~5iF*%=?_Q>} za0OK@J0_0U19nB#{*_c3tSaOiVRJ@Gf7DrGx_*j<0Pr@#LO?M`Q|Z%|C$p>SvLqKA=DtvgWcwJuE79i5-aQD*!iEKh%@#c>2IPtJDJbhsF$F%Z;|{Z2@g4r*W~h zcQL2;154(IM)OzQi{?FW)6)Cry&?0StL|6aLCv0*DVSOHCS=@$XL0US>?Rs6Y!E+* zP3B)lmM0EXF9BO*jq_CI`y2uv99iHN|2V$ME+}E?Jfu9iDNuk z6j)36IA^RS5R;C|Ti+TvHhS@WFqUXNawJ6yOSr`~ZT3|@UY=^XN+^3T%9ep??Ta$4 za?~zHJp_iDQG(cFm|YR~;t*$n8S5^|LP_~>Kx$PYdwzUf%S$N_Ux0_ya#)OYjROgO z(WB_}sPz;bQYiFIeU$Bg)*`9;mt;o){b*kg%1*`EekDYm_1iRrg;2>NBE*gpO)cKf+}7soeKS>A52Vbprd$eeBTF^%w)InMBs zT9>?(O(iEVzEF+oZNu25!knE~!f}WG{P3kAa_<8R`?#J{P#>FP>f;f9*nQSXZp5{* zXM*PW6{2ug02TN)W6bVzi1Rn;=^E82DK`Z#AX=*1(B~&`+>Q>`UL%t{`J;>^ot~r} z2JVP?fVq_H2g}=a$vH9!+bZ~@oIG_NhxX8@i5Q06bij^l42O0&1aE3%9=4?*u7DLx ztaO4h^%}lSJhWp1W@IRq4^S~`;e!Hy#gyqxlX@A6-NZg%`+D>erX?M#x9jXMjvN}4}|Lvf@MxlnQ8wZtq?4;HW?h8Nk=TR6-6_a z)%G2Wf8g~jza9#^8dFrU3O&|>sW^x|@7OCQOWpu3S*xQH7c;}j1q;aq;pE1J@5q~TjtB(&)x-XYi7Z>jb0V}y`^@@mdk2rTi)vv z*A95Qp`L4O2{yEQm4Y)Af9b8NoAAI^yGE_G1xhl{_fFm(xSJlV=||fR$};a|miUi^ zGHZhwb%9}0Z*2hYooc?H%H?#(KGCAq_o?ci{xSnV`t+B%Tt>I-*B|4+bhsgx3l#~B z_;%m3X8GC$w=BNn!Af`qoyQ?c?P2S#1?#TgtiX1uL-z2KRFuTPu|j1&bUy>6_@k`u zTID-jC53frgnwDB+Fh&q%ZAeKM$He?Y7qaC+}z!){*krah#x<0q}M-g*6*&;zAMk) zU8;T8Mo-=?Rne0w1;uOiySK02?nERUQ{iXy0E%hz$^6C}C_Bb)q^Ta)dbwo1 zSk5OI1T7;zf6W!|NxaWm^AbN-y@xcj_KNe4kfoA-L9d0$NOvN(cF82HDIK&pkX~Gv z zFa^&h1q(QmR)rJK-ZPeghQ=4h{LjoO-!i;um_PjeXWo8hsbSCEtfhuS!TJFZ)5EeN zgc(KH>XuskbF2!9=cW~fjYUCY(XC6XiVBvt2Gh4Ln%Zaq312^X?WDgelvXv1+Rb^4B!?)422^SlMcm!93)_vLdp@8h)rTk;tDPCNsWkBxQoxA|1ShFEn20LBK(EU zLH59k$TTo|wM*<`jz0=?XS$2qvWSd#6Y>j3U_>-+W5ef9Ifl1K%0>FMiXJ8?l%fgW zrNKv>#56a*VSey0wy@!L8_NWEHkG2=wrv~M0vIuch>`;15OOu|bX3~#hX40S#NVP| zfPw%8w-H3t_O3&R_tEL%jGl>`^rKF1=><0Bje8NUKn6N=RgbOb1 zR(T>2`=uCm$80H;$xTf3_(SBlEfe~AtylSBOWSMD`#S?!-#rk@XbPIT?i3?nJtT-z z#Nx0ySMclj7X6&3;B5-%vJSCzz%+icaeLK0%o#C^{pub37?jqc9P6#;oyF@;JN{J?i`&A!a@bVV+9z z_bqJhz3m7L`wRUR9Pd8tKlJUX(B`O~^GoTO@Sv*|yp<5A5K&(k9-G7=35cPFGL&Auu%UC&*fpbsNa!_wN8#^$UF&-;P!~+uvZ!e#InkKu zv&>BR4*D+mPvJPI)IiJJu0ZYFvowL9nmOl7^)>q&eKr2!K+>Gt|NPvBxx@2~b9?89 z?*dVSxVIau5@*Z$_neEFY@ z-DwPN>b^5f=_X%o^_`;9zSD+M>X#aJg^D}w>Q($A)59|i~ ztyb@wJ2+o8-}{{wXi0=Za7WR{M(f-t*peQ3{*h=!-|MFUEIsacrN3$BM{GP;|ET)n|C$LPsf!sX1r?n#cJ(0s7(gdXiNPH6pecmnd z7HZb$5r&Kf9zw&APIfe-(6Agy>5H;8A=IgPoC6bcW~z?56tOt@9&(y(Yt(7xKzq?a z9AIkE4Dt4X8xIrBXk;6K_GleIiO*PzwDrn;lE>q5 zVnWsNZTLFtb}l5ilZvJF=L0n9UouTUMS($;m8W(M7$1pQuFwWbf;`@`Ja#B+V22i~{T#N7LT!+!xa4jjS;^mQa8 zqb@vzdy?AWsab(svx0bBlXf7MrG?6|F#x^SG8BQe|VR8AY7_UzakG*#o)7Rh9z?;X-01u^d#4|@&bX0D=U52%={#pPmGBpiO znqg!6*s2jj!Ty3g=`IruMOqL(vlfDnzwZ#@+3ucxV?FW8BbF2!vIMuFTSW>h`;0Xn8TPk}UNnNB1ExzgMq zZEDQ*?~>k>C#d_}<5`|Usdijml}Odq2{^z#stEBFea+BLsYN&qi*cHi1UvglZ;?O1 znB6B~*cCdypi>OyW6pRdXFFWOqV-wE`2LjF>+?f}m|wj8_*;`kV)N!f;}B%aUq|hr zl1xlOX{28ezb6bE|15dWk#~tax=ti@U#LO63@&e7KKVDo zWsv_V@&g;#|@spb7)ta{=B)vHH;&n9reNp6SMb*H_7;iMLQU4MhQfwm^C z>jse>P+vq@umo=}nh%7&0E?{G>_q2rTa+N?)dGqDwHLIb-wnj4fKmxwl5HD-GR@M|NYeCi*jq#F( zYb6b_k|r^3>$S42v9fJz23;Wuh(i{iN!FuDFYrWbq6;e88!z4#E#4L@{#?BH{%G<2 zvEnXDaRw^)CofONE4D@}w#F*9MR)hD6dzry%V$<^8#F+Q+A5>A%DAmLYHJQ3iP?6| zF%!JNs=g2)%R>vb;V-QeZ(ZzOE#9?a+)WmD&Ci(UyW<6Q(E^xh+pL`X7miTNijah^uFTN8QL0{f;+mdC z_6L!J$fRiwS9#DOJlQIK3|JNQ7$y2T%S~*?GNhptfD-Qlorv`U(b)<#Wg7C5Y?2y@ zb0U*atH%c9oL!C1K`V0+woWID7`9IMk-|=Dar4NRc6T{MK7r(r!cV0sQilu@N|`y- zT&!-5Iu+td)Nv@iv|X-UL0)G5UY;^FQ*$N?jb~HFk(ZgefmW58iKo;uoM}#+Y31vV z3d}~~s3275r;&(2Dy~C(iCXDN)Ji}s;fANle1^z;YHG5}r;idnt)V+2w8bvZI!C!!XgdpwTTbIzN%*o(P)V#1~BGE%=z( z0rDK08Z@zB7{f&9JOvy+FR@fWO+sJ-%5lz%Cp=z=9{>+QUhTZVzKq`_hK9tX;|5Pr z2afA}u2Va1UsN7@$&zPs^ou8U+x#={T;k@xJUu5RVAHTnLis)kq&$@ z{vW6a`YgdLj2uxBgIbJ=a5l5)@k4|OVunn4hRniH5+9;LVMledN@NgUVjQ}h0mLJ3!w5lQz$L@2g-)xBlg`VK_j z0Y?W8=FiXRZrWT61y`LfIpbCB(W>@{Yg@#!Jz=pWY;+FdaGw9-0vEAVCv2rJ*5j-) zEgp!K?TVD{j#xUtq8!SSxW%Q4m3T!F@9V@6Ret2woC zEx)hRH~2DnTKW(Sh$B{N0e3-=mjK-n#CxpK$y+F?l{}*6^K`)QMoLAd3I#Q~o9T^l zCJ`r*Ac}c_-k1R9K6+y^nqBk;X(Cz3}c`F|U~en*qaBCE|QCk(KrDqv$Tl?^*^DGThErA)B5m1=^Com4Ja zLP-^nd5Ba26zZpJq>((84Qcq9wwu%q^J8cDT4d!JYV#ZP zMqB)y_|)kNRfB3|^Fq5hU^L;)Ke9%TNDWL7;>jAhZVM!!gte~>s7?Sv_$jzdU=ZZP zs6Wi9Qp)HNDUKx3rRyNkb26b`Je;CRS20t+p3Rk+W)^E?YCdK9uBs*&ThDY1E7D!k z!AqdiPV|-B9ZLf^1^Vy3(^-(tx`A{yq_b}zT`tn)Z6KZF4!xBTOmc7YH<}V)Q31fr zd-u5zZ7kYAx?)d>r_@t+&#~Twbj}T=D@VN*8>qJu>0BE~SA}%d8%S4!_SJ5neRU{r z^9IVR_cTC?uhG-A^Imhhc~C~%J+ASzz(D!Ea=6jf)(y0^O;xhavtk?E-oP2&AErNl4<6B;W|F{zB10&@8^R&YqqO)`&k`KsR|^4Wr&u!+^HPG%`y} zX&NCkrwh#byh(!?4Xl%Q(ihBjy9p}h_n~RQhw0T1$d@4R78lrdPY=8OUZ@!-^b<7V zpPqio%LMYKMu+)P*bkeY8fzg95%nOE)~l)(QUXF}35mN)kBcy$4G%Cnb;CP0&Cd#e zeRWzz)NqQP@UpfLvMs1bJBP=DThfyv(7(q;NhCPmkL)}@OZ?Ozolpl!;xO2hp3{47 z7`oF$-4d>;;nPBG=sD#fcC}#h2J^7QMr2}=iop`$DUPFRR2cI$bp;ok;2yf=h#D?1 z;@)#!egtYeRmlV6xB`KRO}Ln}kp7J+SAW+)Pb02BZYFF=B_&OMa9u;KB|qa0ZbAE9 z^%#LMF&i_6%}`|Zt|z^iQdFjYdh!en5OQPIWn&;P;e@Y)){RXBg4@{LC;++6nXz%E z`9m71lLsDzst|vG3~o=KVo}396mXvL(zroTG7#K3jY|_hA?gy5W~VsuV22x6)&`%K zcQXx`!tb*vvJqE2OgH-O!RB>E2!ZNMVu+9p^->dp?daZ!punV_Mwc#f?%{gafk68L z-VtJ1Ct*`kk~%A>Oy&QQCVnMJx?-e8_-Q=pYp@iQpA9oL%r`yqls6D;qUQ4xumXW0 za|wZ%MSkdFp^R+UFzcyq{>Lbrr+Z7VslUHVl}c%y+vh_Ap_|3$#cG9EEmkUTq_v_f z7<7*q@Pp9zPp$l->loD}m7!jC)ajA&>1k+xZRFpiip#~GO-;K;n7WyDS!o_oi-qzx z(L$e`^g_KX!u+b4xV_1!CJe7GogPSo=eZpLl z)Yn43foA!s2_ADosmc-1^(X;W*fABb`!E9_w02Sd)C37lNZWy*rt&0B#KG zP)@}16RDMU=23puO?yYXk^vaxDVB!1C~P-uLzx=&w=mqs)N}xfYsVTbNh204v*wVr zb@5~9)Ks88Y!zE8?*+K4s-`r>pFWvcAAakG5DX50^n$|7RIf$ z*Q~WS3O2p+aNK<&>OKK+WLx&dLlB_Qo8!9TsIE9$J)AD1hyF7{V{g7DDld%SU zH+NHK5>pt%HLo_j-0=E+v8FEEh%5GUVzMk;x70;h!cQj3yW}zssLC*e9f|Vea?}Y` zlrdbLDDRb{j-*FHnXcHVK2?+{eBjldmwR468Ee^_sNA`vB}=jI6vcM+M0WPRn-?jE zl!ln?s45%UL0xDPBeV3)>JC`LlXq&RA#6#Md*m_)RAsUbz;KXQ-NULVQ+R)(JaY$F zt`l;u2UWRp!YzsNPC06yDvA}Dd9Vy&jjQ0iZrlhQ*{ z6-aa9Q}Si|^e3~yQVg24&v4Q(NSG+*NFrC{Y{LFKNJt{)U_XT<7C}+TM7dD%OJj(% zDk+n3j%vkt$X73^Tn+fVm^u?+#lViL)F(@*j1!WBs7h&0>iy_WmQ$AEP$~YC)xk)u z0%}50po$!*QO*ZikStrONR+aE$mLXOhyzPeOfv?qjCExg24y~tepgz9@e>ydYYppzShc1j=2MKABW~l6!$ZnKRE#lq1X1 zVyv{7A%BJ$0tK>}I78$XVWvO{kR@{}Y8%whT74!}yKG3CfrA7svm*vGKiwfF7crQ> z=?*b@h#_QNnzLw5nh>jNH%ViQKGCf2rdhAp=y<5~p>vW|VxL*nO2=TkRAeSuC~#+( zrIJxkK3b81!$Vqqo&ty(6t0uJIwY*dBd-fjnp=aI&4POF66VEwDj+mVw9&; zwk!#(XY#)UIWY(MUyP+wiMS;%2>9At7L@=e0QCm0Sx^l)J6tebl(fO zwg&AD0tX6!Rv31GNN}zQa=Ij77GlLZLK{H0Bq0y<6f_D%Bsh)Slx7WlVcqqiP-V5^_?^VjH&6j9(>>m5Qz zfaTz4NyDdLFoIAoDrOQV{d}9eBzZK~1e`yk68tlMLIRDzVSx*U5jDO=iM~zV74nFR zA+Y;n){0iqpf$+Ld--Jq5C;sHd0+!$7Qg&-t|$V^B&RG0DeF!TPfYQfP!l`EkKqNN zwHeIfG4rO2-D_FR`urP?icsLn`OD{Hj+QyY4QEq$XUw@9GFo%Sn{|y>AARZ3c-@X@ z-Ht{7O5L7VUT?H+&jR<>**DI6E0%_j>_sG#W`9K+e&CNS3l|no zM2dE<l3ZpUY&e(GCUco3LU)C zceyXr87*mA&YIU#Q?v*>eED!_?*b1)7fmlWg`JBh7H58R;k65q9fu%a_W)OxD672E zbh#_;pzIYZt?T6vVHRhNGf^%i!ieI2f=X7_r4OTXiGR@ z!aUyC5pC>j$)ZM~+V_EwIjtol|ixMe?fpvU2wn4@*h zkgykB`tmbhe)hK_M(1yS1tOQ@FhrrdIgJH3rYwCP$q5)j+WDEe=^q>*yCAE^wh!&O z|8c|Yr1*Tqt74LN2S|GQ{D1ckW9B_~e{YlKZA)=)o$l?10`m9T5%SaAt^z!MR<|Ag zJDUvT*KC6S*P3qn@m>!7c&|=Rev|n~w)VZxRU9tW{=C$J_n&We!vBSyga4VyQNlX= z2HGl6qX3iM8}7Y?Q$eOi<#NThH1Hxt4Quoa53rw~27HNCq#~kE+{%xxK16u4N6cB;h10yo*U^W#uUK% zCV+gizFtpU)F#%{+#!S|xH_oXpFf!l=T)Qz?h?A2JJko_c zGcwLl=N&H6u=fjel^@0I2s*%EGUg4mi7bX~$PmmG!+_qK8KV$J=??BkIrt&sb2}KS z5BNj`wYUiPq(av+d^sUdfofxUfU{yqT`dd1o=Wf`^!cfoiJtfTJ^_3QFP@Pem&4i#a+YmQDx~&zY%&3r}CTc;SZ8amn^g8ms0ldHQ<#tI*r%So6Egv)l^T(M%VNaQ=OS)6nxJ{0?hRX6!qSh!%MLxrSW|LmTKPOt?FPTe6ocZY^Lm{`9Ss({sz%nqnAj^*^waoE?03VGpiH-1$6je*3y;H#7nTCc0{Ut*~ zJcd)l=fF=o6oo3SqCOS=D&=P!vjy}Xo zn6QI{vuEhV00lD$jacRQqvV|;@2li7nC|=Z#$dI#>Fqu8aBtJ_ACvb9c?`Ko5D=SZ zj4LFmBb=%LfHAH!2PGuV3`nz{hT-K&Zzu0Y7PM94Ck_xCvK)6ySIp^B?HaD;mm2$Z zP3d(_;dPDqx+dp3{j^-ySlM6Rb&ct|CKpj3X&OG%l-xAdUDPLX>n>Uo1!RV)4BR!F zmue#Uoe@oLBB$wMcEZto(Vi%1jcD>dz*H)E_E_9f9ko=)EVXe9piRz`Kj|<6<40>;f!2oswkK1;!<0x5ZS|J!*^731KcNvITL67r1n%c zcNjwQ?2#I@f)68gjsxN+^#q5*aBZ_jXPFDGXiF11!_z%q>v?8izUI3P&o_kHAz5dw Pd|&5+aGkD pkh chlog --rebuild -> pkh build -> pkh deb +# +# The catalog deliberately alternates between debian and ubuntu within +# every category, so a default run exercises both archives, series +# namings and chroots. +# +# The file is plain YAML but is also parsed by a small built-in parser when +# PyYAML is unavailable: keep to the subset it supports — full-line comments, +# 'key: value' scalars, inline [a, b] lists, no nested mappings, no block +# scalars, no colons inside values. +# +# Fields (all optional except 'name'): +# category free grouping label (smoke, server, desktop, ...) [defaults.tier] +# tier smoke | standard | long — 'long' is only run when asked for +# dist debian | ubuntu [defaults.dist] +# series pin a series (default: the dist development series) +# arch pass '-a ARCH' to 'pkh deb' +# cross true -> also pass '--cross' (needs 'arch') [false] +# pocket pass '-p POCKET' to pull/deb +# ppa inline list of PPAs passed to 'pkh deb --ppa' +# timeout per-stage budget in minutes [tier default] +# xfail known-broken marker: the reason it is expected to fail +# notes free text, shown by --list and in reports + +defaults: + dist: debian + tier: standard + +packages: + # --- smoke: small, fast, reliable; the first line of defense ------------- + - name: hello + category: smoke + tier: smoke + tags: [gnu, autotools, tiny] + notes: the canonical packaging smoke test + + - name: micro + category: smoke + tier: smoke + tags: [go, editor, tiny] + notes: Go build, golang build-dependencies resolved offline + + - name: jq + category: smoke + tier: smoke + tags: [c, autotools, tiny] + + - name: htop + category: smoke + dist: ubuntu + tier: smoke + tags: [c, autotools, tiny] + + - name: tmux + category: smoke + tier: smoke + tags: [c, autotools, tiny] + + - name: tree + category: smoke + dist: ubuntu + tier: smoke + tags: [c, make, tiny] + + - name: figlet + category: smoke + dist: ubuntu + tier: smoke + tags: [c, make, tiny] + + - name: dash + category: smoke + tier: smoke + tags: [c, autotools, shell] + + - name: file + category: smoke + tier: smoke + tags: [c, autotools] + + - name: zlib + category: smoke + dist: ubuntu + tier: smoke + tags: [c, autotools, lib] + + - name: xz-utils + category: smoke + tier: smoke + tags: [c, autotools] + xfail: sid carries the 5.8.1-1+deb13u1 trixie upload whose packaging fails at dh_installdocs (doxygen api docs missing) - package-side, under triage + + - name: busybox + category: smoke + dist: ubuntu + tier: smoke + tags: [c, kconfig, static] + + - name: dnsmasq + category: smoke + tier: smoke + tags: [c, make, network] + + - name: memcached + category: smoke + dist: ubuntu + tier: smoke + tags: [c, autotools, server] + + - name: kbuild + category: smoke + tier: smoke + tags: [c, kbuild, weird] + notes: the virtualbox build system; exercises the kbuild log classifier + + # --- core ----------------------------------------------------------------- + - name: bash + category: core + tags: [c, autotools, shell] + + - name: coreutils + category: core + dist: ubuntu + tags: [c, autotools] + xfail: pkh deb builds as root in the chroot and coreutils configure refuses root (needs a non-root build user, as Launchpad buildds use) + + - name: util-linux + category: core + tags: [c, autotools] + + - name: openssl + category: core + dist: ubuntu + tags: [c, perl-build, lib] + + - name: vim + category: core + tags: [c, autotools, editor] + + - name: neovim + category: core + dist: ubuntu + tags: [c, cmake, lua, editor] + + - name: git + category: core + tags: [c, make, vcs] + + - name: rust-ripgrep + category: core + dist: ubuntu + tags: [rust, cargo] + notes: debcargo-style vendored Rust build (binary name is ripgrep) + + - name: make-dfsg + category: core + tier: smoke + tags: [c, autotools, tiny] + notes: the GNU make source package (binary name is make) + + - name: systemd + category: core + dist: ubuntu + tags: [c, meson, init] + + # --- server --------------------------------------------------------------- + - name: nginx + category: server + tags: [c, autoconf, server] + + - name: apache2 + category: server + dist: ubuntu + tags: [c, autotools, server] + + - name: openssh + category: server + tags: [c, autotools, server] + + - name: postfix + category: server + dist: ubuntu + tags: [c, make, mail, server] + + - name: dovecot + category: server + tags: [c, autotools, mail, server] + + - name: samba + category: server + dist: ubuntu + tags: [c, python-waf, server] + + - name: bind9 + category: server + tags: [c, autotools, dns, server] + + - name: haproxy + category: server + dist: ubuntu + tags: [c, make, network, server] + + - name: exim4 + category: server + tags: [c, make, mail, server] + + # --- databases -------------------------------------------------------------- + - name: redis + category: database + tags: [c, make, server] + + - name: sqlite3 + category: database + dist: ubuntu + tags: [c, autotools] + + - name: postgresql-17 + category: database + tags: [c, autotools, server] + notes: long-ish but reliable; a good mid-weight benchmark + + - name: leveldb + category: database + dist: ubuntu + tier: smoke + tags: [cpp, cmake] + + - name: mysql-8.0 + category: database + tier: long + dist: ubuntu + timeout: 240 + tags: [cpp, cmake, server] + notes: Ubuntu-only source name; huge C++ build + + # --- desktop ---------------------------------------------------------------- + - name: gtk4 + category: desktop + tags: [c, meson, gui] + + - name: vlc + category: desktop + dist: ubuntu + tags: [cpp, autotools, gui] + + - name: gimp + category: desktop + tags: [c, meson, gui] + + - name: inkscape + category: desktop + dist: ubuntu + tags: [cpp, cmake, gui] + + - name: mutter + category: desktop + tags: [c, meson, gui] + + - name: pipewire + category: desktop + dist: ubuntu + tags: [c, meson, audio] + + - name: plasma-desktop + category: desktop + tags: [cpp, cmake, kde, gui] + + - name: gnome-control-center + category: desktop + dist: ubuntu + tags: [c, meson, gui] + + - name: ffmpeg + category: desktop + tags: [c, autotools, media] + + # --- toolchains --------------------------------------------------------------- + - name: binutils + category: toolchain + tags: [c, autotools] + + - name: gdb + category: toolchain + dist: ubuntu + tags: [c, autotools] + + - name: cmake + category: toolchain + tier: smoke + tags: [cpp, cmake, bootstrap] + + # --- boot / firmware / weird ------------------------------------------------ + - name: dracut + category: boot-firmware + tier: smoke + tags: [shell, make, initramfs, weird] + notes: unusual make logic; initramfs generator + + - name: opensbi + category: boot-firmware + tier: smoke + tags: [c, make, firmware, weird] + notes: RISC-V firmware, cross-built inside a native build + + - name: grub2 + category: boot-firmware + tags: [c, autotools, bootloader, weird] + notes: multi-platform grub; heavy but dependable + + - name: u-boot + category: boot-firmware + dist: ubuntu + arch: arm64 + cross: true + tags: [c, kconfig, firmware, weird, cross] + notes: cross build for arm64; many board configs + + # --- long: hours each; opt-in via --tier long or --tier all ----------------- + - name: gcc-14 + category: toolchain + dist: ubuntu + tier: long + timeout: 360 + tags: [c, autotools, compiler] + + - name: glibc + category: core + tier: long + timeout: 360 + tags: [c, autotools, libc] + + - name: rustc + category: toolchain + tier: long + timeout: 360 + tags: [rust, compiler] + notes: bootstraps the Rust compiler; the longest single build here + + - name: libreoffice + category: desktop + dist: ubuntu + tier: long + timeout: 480 + tags: [cpp, make, office, gui] + + - name: firefox + category: desktop + tier: long + timeout: 480 + tags: [cpp, rust, browser, gui] + + - name: qt6-base + category: desktop + dist: ubuntu + tier: long + timeout: 360 + tags: [cpp, cmake, gui] + + - name: linux + category: kernel + tier: long + dist: ubuntu + timeout: 480 + tags: [c, kbuild, kernel, weird] + notes: the Ubuntu kernel; multi-hour, many flavours, kbuild quirks apply + + - name: openjdk-21 + category: toolchain + tier: long + dist: ubuntu + timeout: 360 + tags: [java, make, compiler] diff --git a/tests/e2e/run.py b/tests/e2e/run.py new file mode 100755 index 0000000..c046c5c --- /dev/null +++ b/tests/e2e/run.py @@ -0,0 +1,931 @@ +#!/usr/bin/env python3 +"""pkh end-to-end package harness. + +Runs real packages from catalog.yml through the full pkh workflow: + + pkh pull (from the archive, by default) + pkh chlog --rebuild (no-change rebuild entry; EDITOR is stubbed out) + pkh build (.dsc + _source.changes next to the tree) + pkh deb (binary packages in an ephemeral chroot) + +Every stage is logged per package, timed, and checked for its artifacts; +results land in /runs//results.json and results.md. Pulls run in +parallel (network-bound), the build pipeline runs sequentially (one heavy +build at a time, and pkh's shared chroot/apt/ccache caches stay warm). + +Usage: + run.py --list + run.py --dry-run + run.py # smoke + standard tiers + run.py --tier long # the multi-hour builds + run.py --only hello,dracut # exact names + run.py --resume runs//results.json # continue an interrupted run + +See README.md in this directory for the full documentation. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import datetime as dt +import json +import os +import platform +import shlex +import shutil +import signal +import subprocess +import sys +import threading +import time +from dataclasses import dataclass, field +from pathlib import Path + +TIERS = ("smoke", "standard", "long") +DEFAULT_TIERS = ("smoke", "standard") +TIER_TIMEOUT_MIN = {"smoke": 30, "standard": 120, "long": 360} +STAGES = ("pull", "chlog", "build", "deb") +PIPELINE_STAGES = ("chlog", "build", "deb") +TERMINAL_OUTCOMES = ("pass", "xpass", "xfail") +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT_DIR = Path(__file__).resolve().parent + + +class CatalogError(Exception): + """Invalid catalog content or unusable options.""" + + +@dataclass +class Entry: + """One catalog entry: a source package and how to build it.""" + + name: str + category: str = "misc" + tier: str = "standard" + dist: str = "debian" + series: str = "" + arch: str = "" + cross: bool = False + pocket: str = "" + ppa: list[str] = field(default_factory=list) + timeout: float = 0.0 # minutes; 0 = the tier default + xfail: str = "" + notes: str = "" + tags: list[str] = field(default_factory=list) + + def timeout_min(self) -> float: + return self.timeout if self.timeout else TIER_TIMEOUT_MIN.get(self.tier, 120) + + +# --------------------------------------------------------------------------- +# Catalog loading: PyYAML when available, a schema-specific fallback parser +# otherwise (the subset it accepts is documented in catalog.yml). +# --------------------------------------------------------------------------- + + +def _scalar(text: str): + text = text.strip() + if text.startswith("[") and text.endswith("]"): + inner = text[1:-1].strip() + return [v.strip().strip("'\"") for v in inner.split(",") if v.strip()] + if len(text) >= 2 and text[0] == text[-1] and text[0] in "'\"": + return text[1:-1] + if text in ("true", "false"): + return text == "true" + if text.isdigit(): + return int(text) + return text + + +def _parse_mini_yaml(text: str) -> dict: + """Parse the flat 'defaults' mapping + flat 'packages' list subset.""" + doc: dict = {"defaults": {}, "packages": []} + mode = None # None | "defaults" | "packages" + item: dict | None = None + for lineno, raw in enumerate(text.splitlines(), 1): + if not raw.strip() or raw.lstrip().startswith("#"): + continue + indent = len(raw) - len(raw.lstrip(" ")) + stripped = raw.strip() + if indent == 0: + if stripped in ("defaults:", "packages:"): + mode = stripped[:-1] + item = None + continue + raise CatalogError(f"catalog:{lineno}: unexpected top-level '{stripped}'") + if mode is None: + raise CatalogError(f"catalog:{lineno}: content outside a top-level key") + if mode == "defaults": + key, sep, value = stripped.partition(":") + if not sep: + raise CatalogError(f"catalog:{lineno}: expected 'key: value'") + doc["defaults"][key.strip()] = _scalar(value) + else: + if stripped.startswith("- "): + item = {} + doc["packages"].append(item) + stripped = stripped[2:].strip() + if not stripped: + continue + if item is None: + raise CatalogError(f"catalog:{lineno}: list items must start with '- '") + key, sep, value = stripped.partition(":") + if not sep: + raise CatalogError(f"catalog:{lineno}: expected 'key: value'") + item[key.strip()] = _scalar(value) + return doc + + +def load_catalog(path: Path) -> list[Entry]: + text = path.read_text(encoding="utf-8") + use_pyyaml = not os.environ.get("PKH_E2E_NO_PYYAML") + try: + if use_pyyaml: + import yaml # type: ignore import-not-found + + doc = yaml.safe_load(text) + else: + doc = _parse_mini_yaml(text) + except CatalogError: + raise + except Exception as exc: + if use_pyyaml: + raise CatalogError(f"cannot parse {path}: {exc}") from exc + raise + + defaults = doc.get("defaults") or {} + raw_entries = doc.get("packages") or [] + if not raw_entries: + raise CatalogError(f"{path}: no packages listed") + + entries: list[Entry] = [] + seen: set[str] = set() + for i, raw in enumerate(raw_entries): + name = str(raw.get("name", "")).strip() + if not name: + raise CatalogError(f"{path}: package #{i + 1} has no name") + if name in seen: + raise CatalogError(f"{path}: duplicate package '{name}'") + seen.add(name) + + entry = Entry( + name=name, + category=str(raw.get("category", "misc")), + tier=str(raw.get("tier", defaults.get("tier", "standard"))), + dist=str(raw.get("dist", defaults.get("dist", "debian"))), + series=str(raw.get("series", "")), + arch=str(raw.get("arch", "")), + cross=bool(raw.get("cross", False)), + pocket=str(raw.get("pocket", "")), + ppa=[str(p) for p in (raw.get("ppa") or [])], + timeout=float(raw.get("timeout", 0)), + xfail=str(raw.get("xfail", "")), + notes=str(raw.get("notes", "")), + tags=[str(t) for t in (raw.get("tags") or [])], + ) + if entry.tier not in TIERS: + raise CatalogError(f"{path}: {name}: unknown tier '{entry.tier}'") + if entry.dist not in ("debian", "ubuntu"): + raise CatalogError(f"{path}: {name}: unknown dist '{entry.dist}'") + if entry.cross and not entry.arch: + raise CatalogError(f"{path}: {name}: cross: true needs an arch") + entries.append(entry) + return entries + + +# --------------------------------------------------------------------------- +# Selection and rendering +# --------------------------------------------------------------------------- + + +def select_entries( + entries: list[Entry], args: argparse.Namespace +) -> tuple[list[Entry], list[str]]: + """Apply the CLI filters; returns the selection and the filtered-out names.""" + tiers = ( + list(TIERS) if args.tier == "all" else [t.strip() for t in args.tier.split(",")] + ) + for tier in tiers: + if tier not in TIERS: + raise CatalogError(f"unknown tier '{tier}' (expected: {', '.join(TIERS)})") + + selected = [e for e in entries if e.tier in tiers] + dropped = [e.name for e in entries if e.tier not in tiers] + + if args.category: + wanted = {c.strip() for c in args.category.split(",") if c.strip()} + dropped += [e.name for e in selected if e.category not in wanted] + selected = [e for e in selected if e.category in wanted] + if args.tag: + wanted = {t.strip() for t in args.tag.split(",") if t.strip()} + dropped += [e.name for e in selected if not wanted.intersection(e.tags)] + selected = [e for e in selected if wanted.intersection(e.tags)] + if args.only: + names = {n.strip() for n in args.only.split(",") if n.strip()} + unknown = sorted(names - {e.name for e in entries}) + if unknown: + raise CatalogError(f"--only: not in the catalog: {', '.join(unknown)}") + dropped += [e.name for e in selected if e.name not in names] + selected = [e for e in selected if e.name in names] + if args.skip: + names = {n.strip() for n in args.skip.split(",") if n.strip()} + dropped += [e.name for e in selected if e.name in names] + selected = [e for e in selected if e.name not in names] + return selected, dropped + + +def fmt_duration(seconds: float | None) -> str: + if seconds is None: + return "-" + seconds = int(seconds) + if seconds < 60: + return f"{seconds}s" + if seconds < 3600: + return f"{seconds // 60}m{seconds % 60:02d}s" + return f"{seconds // 3600}h{(seconds % 3600) // 60:02d}m" + + +def render_list(entries: list[Entry]) -> str: + lines = [ + f"{'PACKAGE':<20} {'TIER':<9} {'DIST':<7} {'CATEGORY':<14} " + f"{'TAGS':<30} NOTES" + ] + for e in entries: + extra = [] + if e.cross: + extra.append(f"cross {e.arch}") + if e.series: + extra.append(f"series {e.series}") + note = " ".join(extra + [e.xfail or e.notes]) + lines.append( + f"{e.name:<20} {e.tier:<9} {e.dist:<7} {e.category:<14} " + f"{','.join(e.tags):<30} {note}" + ) + return "\n".join(lines) + f"\n\n{len(entries)} package(s)\n" + + +# --------------------------------------------------------------------------- +# Command construction and child-process plumbing +# --------------------------------------------------------------------------- + + +def stage_command(args: argparse.Namespace, entry: Entry, stage: str, pkh: Path) -> list[str]: + cmd = [str(pkh), stage] + if stage == "pull": + cmd += [entry.name, "-d", args.dist or entry.dist] + if args.series or entry.series: + cmd += ["-s", args.series or entry.series] + if args.pocket or entry.pocket: + cmd += ["-p", args.pocket or entry.pocket] + if not args.git: + cmd += ["--archive"] + elif stage == "chlog": + cmd += ["--rebuild"] + if args.series or entry.series: + cmd += ["-s", args.series or entry.series] + elif stage == "build": + cmd += ["--verbose"] + elif stage == "deb": + cmd += ["--verbose"] + if args.series or entry.series: + cmd += ["-s", args.series or entry.series] + arch = args.arch or entry.arch + if arch: + cmd += ["-a", arch] + if entry.cross or args.cross: + cmd += ["--cross"] + if args.pocket or entry.pocket: + cmd += ["-p", args.pocket or entry.pocket] + for ppa in entry.ppa + args.ppa: + cmd += ["--ppa", ppa] + if args.jobs: + cmd += ["-j", str(args.jobs)] + if args.no_ccache: + cmd += ["--no-ccache"] + return cmd + + +def child_env() -> dict: + """Environment for pkh children: no editor, a deterministic identity.""" + env = dict(os.environ) + env["EDITOR"] = "true" # chlog launches it after generating; nothing to edit + env.setdefault("DEBEMAIL", "pkh-e2e@invalid") + env.setdefault("DEBFULLNAME", "pkh e2e") + return env + + +def run_stage( + cmd: list[str], cwd: Path, log_path: Path, timeout_min: float +) -> tuple[str, float]: + """Run one stage, logging to log_path; returns (status, seconds). + + Status is 'ok', 'fail' or 'timeout'. The whole process group is killed + on timeout: builds fan out into make/compiler children, and killing only + the direct child would leave them running. On Ctrl+C the group gets a + graceful SIGTERM first, mirroring pkh's own interrupt cleanup. + """ + with log_path.open("ab") as log: + log.write(f"\n$ {shlex.join(cmd)}\n(cwd: {cwd})\n".encode()) + log.flush() + start = time.monotonic() + proc = subprocess.Popen( + cmd, + cwd=cwd, + env=child_env(), + stdin=subprocess.DEVNULL, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + try: + proc.wait(timeout=timeout_min * 60) + except subprocess.TimeoutExpired: + _kill_group(proc, signal.SIGKILL) + return "timeout", time.monotonic() - start + except KeyboardInterrupt: + # A graceful stop first: SIGTERM lets pkh run its own interrupt + # cleanup; a child ignoring it is killed after a grace period. + _kill_group(proc, signal.SIGTERM, wait=10) + _kill_group(proc, signal.SIGKILL) + raise + return ("ok" if proc.returncode == 0 else "fail"), time.monotonic() - start + + +def _kill_group( + proc: subprocess.Popen, sig: signal.Signals, wait: float | None = None +) -> None: + try: + os.killpg(os.getpgid(proc.pid), sig) + except ProcessLookupError: + pass + try: + proc.wait(timeout=wait) + except subprocess.TimeoutExpired: + pass + + +# --------------------------------------------------------------------------- +# The harness proper +# --------------------------------------------------------------------------- + + +class Harness: + def __init__(self, args: argparse.Namespace, entries: list[Entry], pkh: Path): + self.args = args + self.entries = entries + self.pkh = pkh + self.stages = [s.strip() for s in args.stages.split(",") if s.strip()] + self.records: dict[str, dict] = {} + self.pull_results: dict[str, tuple[str, float]] = {} + self.lock = threading.Lock() + self.run_dir = self._open_run_dir() + self.results_path = self.run_dir / "results.json" + (self.run_dir / "logs").mkdir(parents=True, exist_ok=True) + + # -- layout ------------------------------------------------------------- + + def _open_run_dir(self) -> Path: + if self.args.resume: + try: + data = json.loads(Path(self.args.resume).read_text()) + run_dir = Path(data["run_dir"]) + except (OSError, KeyError, json.JSONDecodeError) as exc: + raise CatalogError(f"--resume: cannot read {self.args.resume}: {exc}") + if not run_dir.is_dir(): + raise CatalogError(f"--resume: missing run directory {run_dir}") + for record in data.get("entries", []): + self.records[record["name"]] = record + return run_dir + stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S") + return SCRIPT_DIR / "runs" / f"{stamp}-{os.getpid()}" + + def log_path(self, name: str) -> Path: + return self.run_dir / "logs" / f"{name}.log" + + def workroot(self, entry: Entry) -> Path: + # pkh pull (run from the run root) puts the tree at + # .///: this directory is both the tree's parent and + # where pkh build / pkh deb drop their artifacts. + return self.run_dir / entry.name + + def tree(self, entry: Entry) -> Path: + return self.workroot(entry) / entry.name + + # -- stages --------------------------------------------------------------- + + def do_pull(self, entry: Entry) -> tuple[str, float]: + # Pull from the run root: pkh pull (cwd-less) extracts to + # ./// under its cwd, so every package still gets its own + # subtree and parallel pulls never collide. A failed pull leaves a + # partial tree that pkh pull would refuse to overwrite and that a + # later --resume must not trust: wipe it before retrying and after + # the last attempt. + for attempt in range(self.args.pull_retries + 1): + if attempt: + shutil.rmtree(self.workroot(entry), ignore_errors=True) + status, seconds = run_stage( + stage_command(self.args, entry, "pull", self.pkh), + self.run_dir, + self.log_path(entry.name), + entry.timeout_min(), + ) + if status == "ok": + return status, seconds + shutil.rmtree(self.workroot(entry), ignore_errors=True) + return status, seconds + + def do_chlog(self, entry: Entry) -> tuple[str, float]: + return run_stage( + stage_command(self.args, entry, "chlog", self.pkh), + self.tree(entry), + self.log_path(entry.name), + entry.timeout_min(), + ) + + def do_build(self, entry: Entry) -> tuple[tuple[str, float], list[str]]: + record = run_stage( + stage_command(self.args, entry, "build", self.pkh), + self.tree(entry), + self.log_path(entry.name), + entry.timeout_min(), + ) + artifacts = self.build_artifacts(entry) + return self._check_artifacts(entry, record, artifacts, "dsc/changes"), artifacts + + def do_deb(self, entry: Entry) -> tuple[tuple[str, float], list[str]]: + record = run_stage( + stage_command(self.args, entry, "deb", self.pkh), + self.tree(entry), + self.log_path(entry.name), + entry.timeout_min(), + ) + debs = self.deb_artifacts(entry) + return self._check_artifacts(entry, record, debs, ".deb"), debs + + def build_artifacts(self, entry: Entry) -> list[str]: + return sorted( + p.name + for pattern in ("*.dsc", "*_source.changes") + for p in self.workroot(entry).glob(pattern) + ) + + def deb_artifacts(self, entry: Entry) -> list[str]: + return sorted(p.name for p in self.workroot(entry).glob("*.deb")) + + def _check_artifacts( + self, entry: Entry, record: tuple[str, float], artifacts: list[str], what: str + ) -> tuple[str, float]: + """A green stage with no artifacts is a failure: log it and flip.""" + if record[0] == "ok" and not artifacts: + with self.log_path(entry.name).open("ab") as log: + log.write( + f"\npkh-e2e: stage reported success but produced no {what}\n".encode() + ) + return ("fail", record[1]) + return record + + def changelog_version(self, entry: Entry) -> str: + try: + out = subprocess.run( + ["dpkg-parsechangelog", "-S", "Version"], + cwd=self.tree(entry), + capture_output=True, + text=True, + check=True, + ) + return out.stdout.strip() + except Exception: + return "" + + # -- per-entry pipeline ------------------------------------------------------ + + def run_entry(self, entry: Entry) -> dict: + """Full pipeline for one entry; never raises.""" + # A resumed run keeps what the previous record already achieved + previous = self.records.get(entry.name) + record = { + "name": entry.name, + "category": entry.category, + "tier": entry.tier, + "dist": self.args.dist or entry.dist, + "series": self.args.series or entry.series, + "arch": self.args.arch or entry.arch or platform.machine(), + "cross": entry.cross or self.args.cross, + "tags": entry.tags, + "xfail": entry.xfail, + "notes": entry.notes, + "outcome": "fail", + "version": "", + "stages": {}, + "artifacts": [], + "total_seconds": 0.0, + "log": str(self.log_path(entry.name).relative_to(self.run_dir)), + } + started = time.monotonic() + try: + self._run_entry(record, entry, previous) + finally: + record["total_seconds"] = time.monotonic() - started + if entry.xfail: + record["outcome"] = "xfail" if record["outcome"] == "fail" else "xpass" + with self.lock: + self.records[entry.name] = record + self.flush() + color = {"pass": 32, "xfail": 33, "xpass": 33}.get(record["outcome"], 31) + print( + f"[{entry.name}] \033[{color}m{record['outcome']}\033[0m " + f"({fmt_duration(record['total_seconds'])})" + + (f" — {record['error']}" if record.get("error") else "") + ) + return record + + def _run_entry(self, record: dict, entry: Entry, previous: dict | None) -> None: + # Stages the resumed record already completed are reused rather than + # re-run: a resumed chlog must not bump the version a second time. + # Reuse only while the build selection is unchanged. + reusable = previous is not None and all( + previous.get(key) == record.get(key) + for key in ("dist", "series", "arch", "cross") + ) + prev_stages = previous.get("stages", {}) if previous else {} + + if "pull" in self.stages: + if (self.tree(entry) / "debian").is_dir(): + # Resumed run: the tree survived, keep it (pkh pull refuses + # existing directories anyway). + record["stages"]["pull"] = {"status": "ok", "seconds": 0.0, + "cached": True} + print(f"[{entry.name}] pull ok (tree already present)") + else: + status, seconds = self.pull_results.get(entry.name, ("fail", 0.0)) + record["stages"]["pull"] = {"status": status, "seconds": seconds} + if status != "ok": + record["error"] = f"pull {status}" + return + # Already reported by the pull phase; no duplicate line. + + for stage in PIPELINE_STAGES: + if stage not in self.stages: + continue + prior = prev_stages.get(stage) if reusable else None + if prior and prior.get("status") == "ok": + record["stages"][stage] = {**prior, "reused": True} + if stage == "chlog": + record["version"] = previous.get("version", "") + elif stage == "build": + record["artifacts"] += self.build_artifacts(entry) + elif stage == "deb": + record["artifacts"] += self.deb_artifacts(entry) + print(f"[{entry.name}] {stage} ok (reused)") + continue + if not (self.tree(entry) / "debian").is_dir(): + record["error"] = ( + f"no source tree for {stage}; include the pull stage or " + "resume the original run directory" + ) + return + + if stage == "chlog": + status, seconds = self.do_chlog(entry) + record["stages"]["chlog"] = {"status": status, "seconds": seconds} + if status != "ok": + record["error"] = f"chlog {status}" + return + record["version"] = self.changelog_version(entry) + print( + f"[{entry.name}] chlog ok ({fmt_duration(seconds)})" + + (f" -> {record['version']}" if record["version"] else "") + ) + elif stage == "build": + (status, seconds), artifacts = self.do_build(entry) + record["stages"]["build"] = {"status": status, "seconds": seconds} + record["artifacts"] += artifacts + if status != "ok": + record["error"] = f"build {status}" + return + print(f"[{entry.name}] build ok ({fmt_duration(seconds)})") + elif stage == "deb": + (status, seconds), debs = self.do_deb(entry) + record["stages"]["deb"] = {"status": status, "seconds": seconds} + record["artifacts"] += debs + if status != "ok": + record["error"] = f"deb {status}" + return + print( + f"[{entry.name}] deb ok ({fmt_duration(seconds)}), " + f"{len(debs)} binary package(s)" + ) + + record["outcome"] = "pass" + + # -- run orchestration --------------------------------------------------------- + + def preflight(self) -> None: + version = subprocess.run( + [str(self.pkh), "--version"], capture_output=True, text=True + ) + print(f"[pkh-e2e] pkh: {self.pkh} ({version.stdout.strip() or '?'})") + if "deb" in self.stages: + missing = [ + tool + for tool in ("mmdebstrap", "unshare", "sudo") + if shutil.which(tool) is None + ] + if missing: + print(f"[pkh-e2e] WARNING: deb needs: {', '.join(missing)}") + if subprocess.run(["sudo", "-n", "true"], capture_output=True).returncode: + print( + "[pkh-e2e] WARNING: passwordless sudo unavailable; " + "ephemeral chroot builds will fail" + ) + free_g = shutil.disk_usage(self.run_dir).free / 2**30 + if free_g < 20: + print(f"[pkh-e2e] WARNING: only {free_g:.0f}G free on {self.run_dir}") + print( + f"[pkh-e2e] {len(self.entries)} package(s), stages: " + f"{'+'.join(self.stages)} -> {self.run_dir}" + ) + + def resume_skip(self, entry: Entry) -> bool: + previous = self.records.get(entry.name) + if not previous or previous.get("outcome") not in TERMINAL_OUTCOMES: + return False + return all(s in previous.get("stages", {}) for s in self.stages) + + def pull_phase(self) -> None: + if "pull" not in self.stages: + return + todo = [ + e + for e in self.entries + if not self.resume_skip(e) and not (self.tree(e) / "debian").is_dir() + ] + if not todo: + return + print( + f"[pkh-e2e] pulling {len(todo)} package(s) " + f"({self.args.pull_jobs} at a time)" + ) + with concurrent.futures.ThreadPoolExecutor( + max_workers=self.args.pull_jobs + ) as pool: + futures = {pool.submit(self.do_pull, e): e for e in todo} + for future in concurrent.futures.as_completed(futures): + entry = futures[future] + try: + status, seconds = future.result() + except Exception as exc: # keep the run going on worker errors + status, seconds = "fail", 0.0 + print(f"[{entry.name}] pull crashed: {exc}") + self.pull_results[entry.name] = (status, seconds) + print( + f"[{entry.name}] pull {'ok' if status == 'ok' else status} " + f"({fmt_duration(seconds)})" + ) + + def run(self) -> int: + self.preflight() + self.flush() + + self.pull_phase() + + failures = 0 + for entry in self.entries: + if self.resume_skip(entry): + print(f"[{entry.name}] already {self.records[entry.name]['outcome']}," + " skipping") + continue + if self.args.fail_fast and failures: + print(f"[{entry.name}] skipped (fail-fast)") + continue + record = self.run_entry(entry) + if record["outcome"] in ("fail", "timeout"): + failures += 1 + + self.write_report() + self.print_summary() + return self.exit_code(failures) + + def print_summary(self) -> None: + rows = sorted(self.records.values(), key=lambda r: r["name"]) + print(f"\n=== summary ({len(rows)} recorded) ===") + print( + f"{'PACKAGE':<20} {'TIER':<9} {'RESULT':<8} " + f"{'PULL':>7} {'CHLOG':>7} {'BUILD':>8} {'DEB':>8} {'TOTAL':>8} VERSION" + ) + for r in rows: + def sec(stage): + data = r["stages"].get(stage) + return fmt_duration(data["seconds"]) if data else "-" + + print( + f"{r['name']:<20} {r['tier']:<9} {r['outcome']:<8} " + f"{sec('pull'):>7} {sec('chlog'):>7} {sec('build'):>8} " + f"{sec('deb'):>8} {fmt_duration(r['total_seconds']):>8} " + f"{r['version']}" + ) + tally: dict[str, int] = {} + for r in rows: + tally[r["outcome"]] = tally.get(r["outcome"], 0) + 1 + print("\n" + ", ".join(f"{v} {k}" for k, v in sorted(tally.items()))) + + def exit_code(self, failures: int) -> int: + if failures: + return 1 + if self.args.strict and any( + r["outcome"] == "xpass" for r in self.records.values() + ): + return 1 + return 0 + + # -- persistence --------------------------------------------------------------- + + def flush(self) -> None: + """Persist results progressively: a killed run keeps its records.""" + if not hasattr(self, "_started"): + self._started = dt.datetime.now().astimezone().isoformat() + meta = { + "run_dir": str(self.run_dir), + "pkh": str(self.pkh), + "filters": { + "tier": self.args.tier, + "category": self.args.category, + "tag": self.args.tag, + "only": self.args.only, + "skip": self.args.skip, + "stages": self.stages, + }, + "host": { + "kernel": platform.release(), + "arch": platform.machine(), + "python": platform.python_version(), + }, + "started": self._started, + "updated": dt.datetime.now().astimezone().isoformat(), + "entries": sorted(self.records.values(), key=lambda r: r["name"]), + } + tmp = self.results_path.with_suffix(".json.tmp") + tmp.write_text(json.dumps(meta, indent=2) + "\n") + tmp.replace(self.results_path) + + def write_report(self) -> None: + rows = sorted(self.records.values(), key=lambda r: (r["outcome"], r["name"])) + lines = [ + f"# pkh e2e run — {self.run_dir.name}", + "", + f"- pkh: `{self.pkh}`", + f"- started: {getattr(self, '_started', '?')}", + f"- stages: {'+'.join(self.stages)}", + "", + "| package | tier | result | pull | chlog | build | deb | total |" + " version |", + "|---|---|---|---|---|---|---|---|---|", + ] + for r in rows: + def sec(stage): + data = r["stages"].get(stage) + return fmt_duration(data["seconds"]) if data else "-" + + lines.append( + f"| {r['name']} | {r['tier']} | {r['outcome']} | {sec('pull')} | " + f"{sec('chlog')} | {sec('build')} | {sec('deb')} | " + f"{fmt_duration(r['total_seconds'])} | {r['version']} |" + ) + (self.run_dir / "results.md").write_text("\n".join(lines) + "\n") + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--catalog", type=Path, default=SCRIPT_DIR / "catalog.yml", + help="catalog file (default: catalog.yml)") + parser.add_argument("--tier", default=",".join(DEFAULT_TIERS), + help="tiers to run, comma-separated: smoke, standard, long, " + "or 'all' (default: smoke,standard)") + parser.add_argument("--category", default="", + help="comma-separated categories to keep") + parser.add_argument("--tag", default="", + help="comma-separated tags; keep packages carrying any") + parser.add_argument("--only", default="", + help="comma-separated exact package names to run") + parser.add_argument("--skip", default="", + help="comma-separated exact package names to skip") + parser.add_argument("--stages", default=",".join(STAGES), + help=f"stages to run (default: {','.join(STAGES)})") + parser.add_argument("--pkh", default="", + help="pkh binary to exercise (default: the repo build, " + "then PATH)") + parser.add_argument("--build-pkh", action="store_true", + help="run 'cargo build --release' before resolving pkh") + parser.add_argument("--git", action="store_true", + help="pull through the package VCS instead of the archive") + parser.add_argument("--dist", default="", + help="override the catalog default dist (per-entry dist wins)") + parser.add_argument("--series", default="", + help="override the series (per-entry series wins)") + parser.add_argument("--arch", default="", + help="force the 'pkh deb -a' architecture on every entry") + parser.add_argument("--cross", action="store_true", + help="pass --cross to every deb build") + parser.add_argument("--pocket", default="", + help="distribution pocket passed to pull and deb") + parser.add_argument("--ppa", action="append", default=[], + help="PPA passed to 'pkh deb --ppa' (repeatable)") + parser.add_argument("--jobs", type=int, default=0, + help="pass -j N to 'pkh deb' (default: pkh chooses)") + parser.add_argument("--no-ccache", action="store_true", + help="pass --no-ccache to 'pkh deb'") + parser.add_argument("--pull-jobs", type=int, default=4, + help="parallel pulls (default: 4)") + parser.add_argument("--pull-retries", type=int, default=1, + help="retries per failed pull (default: 1)") + parser.add_argument("--fail-fast", action="store_true", + help="stop the pipeline at the first failure") + parser.add_argument("--strict", action="store_true", + help="also fail the run on xpass") + parser.add_argument("--resume", default="", + help="results.json of a previous run: reuse its run " + "directory and skip finished packages") + parser.add_argument("--list", action="store_true", + help="print the selected packages and exit") + parser.add_argument("--dry-run", action="store_true", + help="print the plan and the exact commands, run nothing") + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + try: + entries = load_catalog(args.catalog) + selected, dropped = select_entries(entries, args) + + stages = [s.strip() for s in args.stages.split(",") if s.strip()] + for stage in stages: + if stage not in STAGES: + raise CatalogError( + f"unknown stage '{stage}' (expected: {', '.join(STAGES)})" + ) + if args.pull_jobs < 1: + raise CatalogError("--pull-jobs must be at least 1") + + if args.list: + print(render_list(selected)) + if dropped: + print(f"(filtered out: {len(dropped)})") + return 0 + if not selected: + print("nothing selected", file=sys.stderr) + return 2 + + pkh = resolve_pkh(args) + if args.dry_run: + print(f"pkh: {pkh}") + print(f"selected {len(selected)} package(s), stages: {args.stages}") + for entry in selected: + where = f" {entry.series}" if entry.series else "" + print(f"\n[{entry.name}] {entry.tier}/{entry.category} " + f"({entry.dist}{where})") + for stage in stages: + cmd = stage_command(args, entry, stage, pkh) + print(f" {shlex.join(cmd)}" + f" # timeout {entry.timeout_min():.0f}m") + return 0 + + harness = Harness(args, selected, pkh) + return harness.run() + except CatalogError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + except KeyboardInterrupt: + print("\ninterrupted", file=sys.stderr) + return 130 + + +def resolve_pkh(args: argparse.Namespace) -> Path: + if args.pkh: + path = Path(args.pkh).resolve() + if not path.is_file(): + raise CatalogError(f"--pkh: no such file: {path}") + return path + if args.build_pkh: + subprocess.run(["cargo", "build", "--release"], cwd=REPO_ROOT, check=True) + for candidate in (REPO_ROOT / "target/release/pkh", REPO_ROOT / "target/debug/pkh"): + if candidate.is_file(): + return candidate + found = shutil.which("pkh") + if not found: + raise CatalogError( + "no pkh binary: pass --pkh, use --build-pkh, or install pkh on PATH" + ) + return Path(found) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))