diff --git a/plans/native-build.md b/plans/native-build.md index b837c91..32c5c72 100644 --- a/plans/native-build.md +++ b/plans/native-build.md @@ -115,7 +115,8 @@ Exit codes matter: e.g. unsatisfied build-deps ⇒ exit 3. | `dpkg-architecture` | arch ↔ triplet tables, multiarch tuple, env dump | **Low-medium** — embed cputable/ostable/tupletable/abitable data (stable for years) | **Replaced** (§11, [`debian/arch.rs`](../src/debian/arch.rs)) | | `dpkg-checkbuilddeps` | deps vs installed status | **Medium** — `Dpkg::Deps` grammar (alternatives, arch qualifiers, `` restrictions, versioned Provides subtleties, Multi-Arch facts) + status-file scan | **Replaced** (§11, [`debian/deps.rs`](../src/debian/deps.rs); wired into the pipeline behind `-D`, source-only builds skip it like dpkg-buildpackage) | | `dpkg-genbuildinfo` | `.buildinfo` | **Medium** — deb822 emit + status snapshot + checksums | Native (see §11, [`buildinfo.rs`](../src/build/buildinfo.rs)) | -| `dpkg-genchanges` | `.changes` | **Medium** — deb822 emit + `debian/files` consumption + `.deb` control extraction (ar+tar, trivial with crates) | Native for source uploads (§11, [`changes.rs`](../src/build/changes.rs)); binary aggregation next | +| `dpkg-genchanges` | `.changes` | **Medium** — deb822 emit + `debian/files` consumption + `.deb` control extraction (ar+tar, trivial with crates) | **Replaced** for source (§11) and binary uploads (§11, [`build/binary.rs`](../src/build/binary.rs)) | +| `dpkg-genbuildinfo` (binary) | `.buildinfo` for `-b` builds | **Medium** — deb822 emit + in-context artifact hashing | **Replaced** (§11, [`build/binary.rs`](../src/build/binary.rs), wired into [`deb/local.rs`](../src/deb/local.rs)) | | `dpkg-distaddfile`/`debian/files` protocol | build outputs registry | **Trivial** — one append-only line format | Native ([`files.rs`](../src/build/files.rs)) | | OpenPGP signing | inline clearsign of dsc/buildinfo/changes | **Low** — `gpgme` (already a dependency) supports clearsigning | Native ([`sign.rs`](../src/build/sign.rs)) | | `dpkg-source` | orig tarball, debian diff, patches, `.dsc` | **Very high** — V1/V2/quilt/native formats, byte-exact tar normalization, quilt bookkeeping, `--include-binaries`, hundreds of validation warnings | **Keep as subprocess** (see §6) | @@ -304,6 +305,24 @@ correct for quilt formats. `Environment` field, vendor default profiles approximated (`derivative.ubuntu noudeb` for Ubuntu). -**Next steps**: binary-build adoption of the same pipeline inside ephemeral -contexts (`.changes`/`.buildinfo` generation for `pkh deb`, replacing the -manual quilt step), then Phase 2 satellite replacement. +## 12. Implementation status (Phase 2 — satellite tools) + +All four satellite replacements landed, each gated by differential tests +against the real tool: + +| Work item | Module | Differential gate | +|---|---|---| +| WI-1 `dpkg-architecture` | [`debian/arch.rs`](../src/debian/arch.rs) (tables + lookups), wired into [`build/env.rs`](../src/build/env.rs) | `arch_env(Some(a))` equals real `dpkg-architecture -f -a a` key-for-key for **every** arch from `dpkg-architecture -L`, plus native (`build/mod.rs::diff_arch_env_*`) | +| WI-2 version compare | [`debian/version.rs`](../src/debian/version.rs) (`Ord`, `compare`, `later_than`) | all vectors from dpkg `scripts/t/Dpkg_Version.t` + Ubuntu-flavored cases, cross-checked against `dpkg --compare-versions` for `<< <= = >= >>` (`diff_version_compare_against_dpkg`) | +| WI-3 `dpkg-checkbuilddeps` | [`debian/deps.rs`](../src/debian/deps.rs) (grammar, restriction reduction, KnownFacts evaluation, `check_build_depends`) | 24 scenarios vs real `dpkg-checkbuilddeps` (alternatives, versions, arch/profile restrictions, Multi-Arch, versioned Provides, conflicts, `-A`/`-B`) comparing exit status + diagnostics (`diff_checkbuilddeps_matrix`); unit tests port the `Dpkg_Deps.t` reduction matrices. Wired into `run_source_build` behind `-D` parity: source-only builds skip the check like `dpkg-buildpackage`, unsatisfied deps exit 3 | +| WI-4 binary `.buildinfo`/`.changes` | [`build/binary.rs`](../src/build/binary.rs) (context-generic generation), wired into [`deb/local.rs`](../src/deb/local.rs); artifact retrieval extended in [`deb/mod.rs`](../src/deb/mod.rs) | same tree built with real `dpkg-buildpackage -b` and with the pkh flow (rules build/binary + native metadata through a local context): `.changes`/`.buildinfo` compared field-by-field modulo machine-dependent fields, artifact checksums included (`diff_binary_build_metadata`) | + +Binary-flow notes: `SOURCE_DATE_EPOCH` is now exported to the rules +environment (reproducibility); digests are computed inside the context via +coreutils so remote/chrooted trees work; binNMU binary builds set +`Source: pkg (prev)` / `Binary-Only-Changes` and redistribute the previous +`.dsc` when present; `Architecture` is encounter-ordered in `.changes` +(sorted in `.buildinfo`, matching dpkg). + +Still delegated to subprocesses: `dpkg-source` and `debian/rules` (by +design, see §5 Scope C). diff --git a/src/build/binary.rs b/src/build/binary.rs new file mode 100644 index 0000000..1e313c2 --- /dev/null +++ b/src/build/binary.rs @@ -0,0 +1,493 @@ +//! Binary-build metadata generation: native `.buildinfo` / `.changes` +//! production for binary-only builds (`pkh deb`), the equivalent of +//! `dpkg-genbuildinfo -b` + `dpkg-genchanges -b`. +//! +//! All tree/database access goes through a [`Context`] so the generation can +//! run against a build tree living in a local directory, an ephemeral +//! chroot or a remote host. Artifact digests are computed inside the context +//! with coreutils (`md5sum`, `sha1sum`, `sha256sum`, `stat`), keeping the +//! flow binary-safe regardless of the transport. + +use std::collections::BTreeMap; +use std::error::Error; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use crate::context::Context; +use crate::debian::{ + parse_changelog_entry_from_str, ChecksumEntry, ControlInfo, FileChecksums, FilesList, +}; + +/// Digests of one artifact. +#[derive(Debug, Clone, Default)] +struct ArtifactHashes { + size: u64, + md5: String, + sha1: String, + sha256: String, +} + +/// Options driving binary metadata generation. +#[derive(Debug, Clone)] +pub struct BinaryMetadataOptions { + /// Active build profiles (`Built-For-Profiles`). + pub profiles: Vec, + /// Vendor name (`Build-Origin`). + pub vendor: String, + /// Parallel job count advertised in `DEB_BUILD_OPTIONS`. + pub parallel: usize, + /// Reproducible-builds epoch exported to the build. + pub source_date_epoch: i64, + /// Build architecture (the machine inside the build context). + pub build_arch: String, + /// Host architecture (the packages' target); equals the build + /// architecture except for cross builds. + pub host_arch: String, +} + +/// Generate `__.buildinfo` and `.changes` for a finished +/// binary build, consuming `debian/files` from `package_dir` and the +/// artifacts sitting in `upload_dir`. Returns both paths (inside the +/// context). +/// +/// Mirrors the observable behavior of `dpkg-genbuildinfo -b` and +/// `dpkg-genchanges -b`: sorted `Binary` list, encounter-order `Architecture` +/// accumulation, sorted `Description` lines formatted like dpkg, `.buildinfo` +/// registration in `debian/files`, and binary-NMU handling (`Source: +/// pkg (prev)` + previous `.dsc` redistribution when present). +pub fn generate_binary_metadata( + ctx: &Arc, + package_dir: &Path, + upload_dir: &Path, + opts: &BinaryMetadataOptions, +) -> Result<(PathBuf, PathBuf), Box> { + // ------------------------------------------------------------------ + // Metadata sources inside the context + // ------------------------------------------------------------------ + let changelog_content = ctx.read_file(&package_dir.join("debian/changelog"))?; + let entry = parse_changelog_entry_from_str(&changelog_content)?; + + let control_content = ctx.read_file(&package_dir.join("debian/control"))?; + let control = ControlInfo::parse_content(&control_content)?; + + let files_content = ctx + .read_file(&package_dir.join("debian/files")) + .unwrap_or_default(); + let mut files_list = FilesList::parse(&files_content)?; + + // ------------------------------------------------------------------ + // Collect binary artifacts registered in debian/files + // ------------------------------------------------------------------ + let artifact_names: Vec = files_list + .iter() + .filter(|e| { + matches!(e.package_type.as_deref(), Some("deb") | Some("udeb")) + }) + .map(|e| e.filename.clone()) + .collect(); + + if artifact_names.is_empty() { + return Err( + "binary build with no binary artifacts found; cannot distribute".into(), + ); + } + + let mut hashes = hashes_in_context(ctx, upload_dir, &artifact_names)?; + + let mut checksums = FileChecksums::new(); + let mut arch_values: Vec = Vec::new(); + let mut arch_seen = std::collections::HashSet::new(); + for name in &artifact_names { + let entry_hashes = hashes.remove(name).ok_or_else(|| { + format!("artifact '{name}' listed in debian/files but not found") + })?; + checksums.insert_entry( + name, + ChecksumEntry { + size: entry_hashes.size, + md5: entry_hashes.md5, + sha1: entry_hashes.sha1, + sha256: entry_hashes.sha256, + }, + ); + // Architecture accumulation in encounter order (dpkg-genchanges). + if let Some(file_entry) = files_list.get(name) + && let Some(arch) = file_entry.arch.as_ref().or_else(|| { + file_entry.attrs.get("architecture") + }) + && arch_seen.insert(arch.clone()) + { + arch_values.push(arch.clone()); + } + } + + // ------------------------------------------------------------------ + // Binary-NMU: redistribute the previous source when present + // ------------------------------------------------------------------ + let sversion = entry.version.no_epoch(); + let mut source_display = entry.source.clone(); + let mut binary_only_changes = None; + + if entry.binary_only + && let Ok(prev_entry) = + crate::debian::changelog::parse_previous_version_from_str(&ctx.read_file( + &package_dir.join("debian/changelog"), + )?) + && let Some(prev) = prev_entry + { + source_display = format!("{} ({})", entry.source, prev); + binary_only_changes = Some(format!( + "{}\n\n -- {} <{}> {}", + entry.changes_field, + entry.maintainer_name, + entry.maintainer_email, + entry.date_raw + )); + let prev_version = crate::debian::DebianVersion::parse(&prev)?; + let dsc_name = format!("{}_{}.dsc", entry.source, prev_version.no_epoch()); + let dsc_path = upload_dir.join(&dsc_name); + if ctx.exists(&dsc_path)? { + include_dsc_artifacts(ctx, upload_dir, &dsc_name, &mut checksums)?; + } + } + + // ------------------------------------------------------------------ + // Binary package names and descriptions + // ------------------------------------------------------------------ + let mut binaries: Vec = Vec::new(); + for name in &artifact_names { + if let Some(e) = files_list.get(name) + && let Some(pkg) = &e.package + && !binaries.contains(pkg) + { + binaries.push(pkg.clone()); + } + } + binaries.sort(); + + // Description lines: first line of each binary stanza's Description, + // formatted exactly like dpkg-genchanges, sorted. + let mut descriptions = Vec::new(); + for stanza in &control.binaries { + let Some(pkg) = stanza.get("Package") else { + continue; + }; + if !binaries.contains(&pkg.to_string()) { + continue; + } + let summary = stanza + .get("Description") + .unwrap_or("no description available") + .lines() + .next() + .unwrap_or("no description available"); + // Package-Type overrides the artifact-derived type (deb default). + let pkg_type = stanza + .get("Package-Type") + .map(str::to_string) + .unwrap_or_else(|| { + files_list + .iter() + .find(|f| f.package.as_deref() == Some(pkg)) + .and_then(|f| f.package_type.clone()) + .unwrap_or_else(|| "deb".to_string()) + }); + descriptions.push(crate::build::changes::format_description( + pkg, &pkg_type, summary, + )); + } + descriptions.sort(); + + // ------------------------------------------------------------------ + // Installed-Build-Depends closure over the context status database + // ------------------------------------------------------------------ + let status_content = ctx + .read_file(Path::new("/var/lib/dpkg/status")) + .unwrap_or_default(); + let bd_fields = [ + control.source.get("Build-Depends").unwrap_or(""), + control.source.get("Build-Depends-Arch").unwrap_or(""), + control.source.get("Build-Depends-Indep").unwrap_or(""), + ]; + let installed_build_depends = + crate::build::buildinfo::installed_build_depends_from_content( + &status_content, + &bd_fields, + )?; + + // ------------------------------------------------------------------ + // .buildinfo generation, then registration in debian/files + // ------------------------------------------------------------------ + let pipeline_env = pipeline_environment(opts); + let environment = crate::build::env::buildinfo_environment(&pipeline_env); + + // dpkg-genbuildinfo sorts the accumulated architecture values, while + // dpkg-genchanges keeps encounter order. + let mut buildinfo_arch_values = arch_values.clone(); + buildinfo_arch_values.sort(); + + let buildinfo_name = format!( + "{}_{}_{}.buildinfo", + entry.source, + sversion, + opts.host_arch + ); + let buildinfo_doc = crate::build::buildinfo::render_buildinfo( + &crate::build::buildinfo::BuildInfoInput { + source: source_display.clone(), + binaries: binaries.clone(), + architecture: buildinfo_arch_values.join(" "), + version: entry.version.full(), + binary_only_changes: binary_only_changes.clone(), + build_origin: opts.vendor.clone(), + build_architecture: opts.build_arch.clone(), + build_date: chrono::Local::now().to_rfc2822(), + checksums: checksums.clone(), + installed_build_depends, + environment, + }, + ); + let buildinfo_path = upload_dir.join(&buildinfo_name); + ctx.write_file(&buildinfo_path, &crate::debian::control::write_paragraph(&buildinfo_doc))?; + + // Register the .buildinfo in debian/files, like dpkg-genbuildinfo does, + // so the .changes distributes it. + files_list.add(crate::debian::FilesEntry::new( + &buildinfo_name, + control.section(), + control.priority(), + )); + ctx.write_file( + &package_dir.join("debian/files"), + &files_list.render(), + )?; + + // Hash the freshly written .buildinfo inside the context. + let buildinfo_hashes = hashes_in_context( + ctx, + upload_dir, + std::slice::from_ref(&buildinfo_name), + )?; + if let Some(h) = buildinfo_hashes.get(&buildinfo_name) { + checksums.insert_entry( + &buildinfo_name, + ChecksumEntry { + size: h.size, + md5: h.md5.clone(), + sha1: h.sha1.clone(), + sha256: h.sha256.clone(), + }, + ); + } + + // ------------------------------------------------------------------ + // .changes generation + // ------------------------------------------------------------------ + let changes_name = format!("{}_{}_{}.changes", entry.source, sversion, opts.host_arch); + let changed_by = format!("{} <{}>", entry.maintainer_name, entry.maintainer_email); + let changes_doc = crate::build::changes::render_changes(&crate::build::changes::ChangesInput { + date: entry.date_raw.clone(), + source: source_display, + binaries, + built_for_profiles: opts.profiles.clone(), + architecture: arch_values.join(" "), + version: entry.version.full(), + distribution: entry.distribution.clone(), + urgency: entry.urgency.clone(), + maintainer: control.source.get("Maintainer").map(str::to_string), + changed_by: Some(changed_by), + descriptions, + closes: entry.closes.clone(), + changes_field: entry.changes_field.clone(), + checksums, + files_list, + }); + let changes_path = upload_dir.join(&changes_name); + ctx.write_file(&changes_path, &crate::debian::control::write_paragraph(&changes_doc))?; + + Ok((buildinfo_path, changes_path)) +} + +/// Environment exported to the build steps; recorded (filtered) in the +/// `.buildinfo` `Environment` field. +fn pipeline_environment(opts: &BinaryMetadataOptions) -> BTreeMap { + let mut env = BTreeMap::new(); + env.insert( + "SOURCE_DATE_EPOCH".to_string(), + opts.source_date_epoch.to_string(), + ); + env.insert( + "DEB_BUILD_OPTIONS".to_string(), + format!("parallel={}", opts.parallel), + ); + if !opts.profiles.is_empty() { + env.insert( + "DEB_BUILD_PROFILES".to_string(), + opts.profiles.join(","), + ); + } + env +} + +/// Compute md5/sha1/sha256 digests and sizes for the named files inside the +/// context directory `dir`, using coreutils. +fn hashes_in_context( + ctx: &Arc, + dir: &Path, + names: &[String], +) -> Result, Box> { + let mut out: BTreeMap = names + .iter() + .map(|n| (n.clone(), ArtifactHashes::default())) + .collect(); + + // Sizes. + let output = ctx + .command("stat") + .current_dir(dir) + .arg("-c") + .arg("%s %n") + .args(names) + .output()?; + let stdout = String::from_utf8_lossy(&output.stdout); + for line in stdout.lines() { + let Some((size, name)) = line.trim().split_once(' ') else { + continue; + }; + if let Some(slot) = out.get_mut(name) { + slot.size = size.parse().unwrap_or(0); + } + } + + // Digests. + for (tool, field) in [ + ("md5sum", 0usize), + ("sha1sum", 1usize), + ("sha256sum", 2usize), + ] { + let output = ctx + .command(tool) + .current_dir(dir) + .args(names) + .output() + .map_err(|e| format!("failed to run '{tool}' inside the build context: {e}"))?; + if !output.status.success() { + return Err(format!( + "'{tool}' failed inside the build context: {}", + String::from_utf8_lossy(&output.stderr).trim() + ) + .into()); + } + let stdout = String::from_utf8_lossy(&output.stdout); + for line in stdout.lines() { + let Some((digest, name)) = line.trim().split_once(" ") else { + continue; + }; + let name = name.trim_start_matches('*'); + if let Some(slot) = out.get_mut(name) { + match field { + 0 => slot.md5 = digest.to_string(), + 1 => slot.sha1 = digest.to_string(), + _ => slot.sha256 = digest.to_string(), + } + } + } + } + + Ok(out) +} + +/// Pull the `.dsc` checksums (and its referenced tarballs) into the +/// checksum registry, mirroring how binary-NMU uploads redistribute the +/// previous source. +fn include_dsc_artifacts( + ctx: &Arc, + upload_dir: &Path, + dsc_name: &str, + checksums: &mut FileChecksums, +) -> Result<(), Box> { + let dsc_content = ctx.read_file(&upload_dir.join(dsc_name))?; + let para = crate::debian::control::parse_paragraphs(&dsc_content) + .into_iter() + .next() + .ok_or_else(|| format!("'{dsc_name}' is empty"))?; + + let mut names: Vec = Vec::new(); + let mut partials: BTreeMap = BTreeMap::new(); + for field in ["Checksums-Sha1", "Checksums-Sha256"] { + if let Some(value) = para.get(field) { + for line in value.lines() { + let tokens: Vec<&str> = line.split_whitespace().collect(); + if tokens.len() != 3 { + continue; + } + let slot = partials.entry(tokens[2].to_string()).or_default(); + if field == "Checksums-Sha1" { + slot.sha1 = Some(tokens[0].to_string()); + } else { + slot.sha256 = Some(tokens[0].to_string()); + } + slot.size = tokens[1].parse().ok().or(slot.size); + } + } + } + if let Some(files_value) = para.get("Files") { + for line in files_value.lines() { + let tokens: Vec<&str> = line.split_whitespace().collect(); + if tokens.len() >= 3 { + let slot = partials.entry(tokens[2].to_string()).or_default(); + slot.md5 = Some(tokens[0].to_string()); + slot.size = tokens[1].parse().ok().or(slot.size); + } + } + } + for field in ["Checksums-Sha1", "Checksums-Sha256"] { + if let Some(value) = para.get(field) { + for line in value.lines() { + if let Some(name) = line.split_whitespace().nth(2) { + names.push(name.to_string()); + } + } + } + } + + // The .dsc itself is hashed fresh (it may be signed/rewritten); the + // tarballs reuse the .dsc-recorded digests, like dpkg-genchanges does. + let dsc_hashes = hashes_in_context(ctx, upload_dir, std::slice::from_ref(&dsc_name.to_string()))?; + if let Some(h) = dsc_hashes.get(dsc_name) { + checksums.insert_entry( + dsc_name, + ChecksumEntry { + size: h.size, + md5: h.md5.clone(), + sha1: h.sha1.clone(), + sha256: h.sha256.clone(), + }, + ); + } + for name in &names { + if name == dsc_name { + continue; + } + let p = &partials[name]; + checksums.insert_entry( + name, + ChecksumEntry { + size: p.size.unwrap_or(0), + md5: p.md5.clone().unwrap_or_default(), + sha1: p.sha1.clone().unwrap_or_default(), + sha256: p.sha256.clone().unwrap_or_default(), + }, + ); + } + Ok(()) +} + +/// Partially-known checksums taken from a `.dsc` checksum field. +#[derive(Debug, Default)] +struct PartialDscChecksums { + size: Option, + md5: Option, + sha1: Option, + sha256: Option, +} diff --git a/src/build/buildinfo.rs b/src/build/buildinfo.rs index c892093..938191d 100644 --- a/src/build/buildinfo.rs +++ b/src/build/buildinfo.rs @@ -28,13 +28,6 @@ struct StatusDb { } impl StatusDb { - /// Parse a dpkg status file (e.g. `/var/lib/dpkg/status`). - fn load(path: &Path) -> Result> { - let content = std::fs::read_to_string(path) - .map_err(|e| format!("cannot read status file '{}': {}", path.display(), e))?; - Ok(Self::from_str(&content)) - } - fn from_str(content: &str) -> StatusDb { let mut db = StatusDb::default(); for para in parse_paragraphs(content) { @@ -132,7 +125,20 @@ pub fn installed_build_depends( status_path: &Path, build_depends_fields: &[&str], ) -> Result> { - let db = StatusDb::load(status_path)?; + let content = std::fs::read_to_string(status_path) + .map_err(|e| format!("cannot read status file '{}': {}", status_path.display(), e))?; + installed_build_depends_from_content(&content, build_depends_fields) + .map_err(|e| e.into()) +} + +/// Compute the `Installed-Build-Depends` value from the textual content of a +/// dpkg status database (used when the database lives in another context, +/// e.g. inside a chroot). +pub fn installed_build_depends_from_content( + status_content: &str, + build_depends_fields: &[&str], +) -> Result { + let db = StatusDb::from_str(status_content); let mut work: VecDeque = VecDeque::new(); for name in &db.essential { diff --git a/src/build/mod.rs b/src/build/mod.rs index 5b4c731..86a92b8 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -6,6 +6,7 @@ //! source-tree work (tarballs, diffs, patches) to `dpkg-source` as a //! subprocess. +pub mod binary; pub mod buildinfo; pub mod buildtype; pub mod changes; @@ -771,7 +772,7 @@ mod differential_tests { if matches!(key, "Checksums-Sha1" | "Checksums-Sha256" | "Files") { let without_buildinfo = |v: &str| -> String { v.lines() - .filter(|l| !l.trim_end().ends_with("_source.buildinfo")) + .filter(|l| !l.trim_end().ends_with(".buildinfo")) .collect::>() .join("\n") }; @@ -1125,6 +1126,114 @@ Provides: virtual-thing (= 2.0), plain-virtual } } + /// Differential check of the binary-build metadata generation against + /// real `dpkg-buildpackage -b`: both sides build the same tree (rules + /// driving dpkg-gencontrol/dpkg-deb directly, no debhelper needed), + /// then the produced `.changes`/`.buildinfo` are compared field by + /// field modulo machine-dependent values. + #[test] + fn diff_binary_build_metadata() { + const NAME: &str = "pkh-diff-m"; + let control = format!( + "Source: {NAME}\nSection: utils\nPriority: optional\nMaintainer: {MAINTAINER}\nBuild-Depends: libc6\n\n\ + Package: {NAME}\nArchitecture: any\nDescription: test package main\n long description\n\n\ + Package: {NAME}-u\nPackage-Type: udeb\nArchitecture: all\nDescription: test udeb\n short\n" + ); + let changelog = format!( + "{NAME} (1.0-1) unstable; urgency=medium\n\n * Binary build test.\n\n -- {MAINTAINER} {DATE}\n" + ); + let rules = format!( + "#!/usr/bin/make -f\nV = $(shell dpkg-parsechangelog -S Version)\nA = $(shell dpkg-architecture -qDEB_HOST_ARCH)\n\nbuild:\n\tmkdir -p debian/tmp/usr/bin\n\tprintf '#!/bin/sh\\necho hi\\n' > debian/tmp/usr/bin/hello\n\tchmod 755 debian/tmp/usr/bin/hello\n\ttouch $@\n\nbinary: build\n\trm -rf debian/{NAME} debian/{NAME}-u\n\tmkdir -p debian/{NAME}/usr/bin debian/{NAME}/DEBIAN\n\tcp -r debian/tmp/. debian/{NAME}/\n\tdpkg-gencontrol -p{NAME} -Pdebian/{NAME}\n\tdpkg-deb --build debian/{NAME} ..\n\tmkdir -p debian/{NAME}-u/usr/share debian/{NAME}-u/DEBIAN\n\techo data > debian/{NAME}-u/usr/share/data.txt\n\tdpkg-gencontrol -p{NAME}-u -Pdebian/{NAME}-u\n\tdpkg-deb --build debian/{NAME}-u ..\n\tmv ../{NAME}-u_$(V)_all.deb ../{NAME}-u_$(V)_all.udeb\n\nclean:\n\trm -rf debian/tmp debian/{NAME} debian/{NAME}-u build-stamp debian/files debian/*.substvars\n\n.PHONY: build binary clean\n" + ); + + let write_tree = |root: &Path| { + fs::create_dir_all(root.join(format!("{NAME}/debian/source"))).expect("mkdir tree"); + let tree = root.join(NAME); + fs::write(tree.join("debian/control"), &control).expect("write control"); + fs::write(tree.join("debian/changelog"), &changelog).expect("write changelog"); + fs::write(tree.join("debian/source/format"), "3.0 (native)\n") + .expect("write format"); + fs::write(tree.join("debian/rules"), &rules).expect("write rules"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(tree.join("debian/rules"), fs::Permissions::from_mode(0o755)) + .expect("chmod rules"); + } + tree + }; + + let base = tempfile::tempdir().expect("tempdir"); + let golden_root = base.path().join("golden"); + let ours_root = base.path().join("ours"); + fs::create_dir_all(&golden_root).expect("mkdir golden"); + fs::create_dir_all(&ours_root).expect("mkdir ours"); + + let golden_tree = write_tree(&golden_root); + let ours_tree = write_tree(&ours_root); + + // Golden side: real dpkg-buildpackage binary build. + let status = Command::new("dpkg-buildpackage") + .current_dir(&golden_tree) + .args(["-b", "-d", "--no-sign"]) + .status() + .expect("run dpkg-buildpackage (is dpkg-dev installed?)"); + assert!(status.success(), "golden dpkg-buildpackage -b failed"); + + // Ours: emulate the pkh deb flow (rules build + rules binary with a + // dpkg-buildpackage-like environment), then run the native metadata + // generation through a local context. dpkg-buildpackage runs the + // rules targets directly by default (missing Rules-Requires-Root is + // treated as 'no'), so no fakeroot wrapper here either. + let entry = + crate::debian::parse_changelog_entry_from_str(&changelog).expect("parse changelog"); + let vendor = env::current_vendor(); + let profiles = env::resolve_build_profiles(&[], &vendor); + let parallel = env::num_parallel(); + let build_env_vars: Vec<(String, String)> = [ + ("LANG".to_string(), "C".to_string()), + ("DEB_BUILD_OPTIONS".to_string(), format!("parallel={parallel}")), + ("SOURCE_DATE_EPOCH".to_string(), entry.timestamp.to_string()), + ] + .into_iter() + .collect(); + + for target in ["build", "binary"] { + let status = Command::new("debian/rules") + .current_dir(&ours_tree) + .envs(build_env_vars.clone()) + .arg(target) + .status() + .expect("run rules target"); + assert!(status.success(), "debian/rules {target} failed"); + } + + let ctx = std::sync::Arc::new(crate::context::Context::new( + crate::context::ContextConfig::Local, + )); + let native_arch = crate::debian::arch::native().unwrap_or_else(|_| "amd64".into()); + let opts = crate::build::binary::BinaryMetadataOptions { + profiles, + vendor, + parallel, + source_date_epoch: entry.timestamp, + build_arch: native_arch.clone(), + host_arch: native_arch, + }; + crate::build::binary::generate_binary_metadata(&ctx, &ours_tree, &ours_root, &opts) + .expect("native binary metadata generation failed"); + + // Compare artifacts. + assert_changes_equivalent( + &golden_root.join(format!("{NAME}_1.0-1_amd64.changes")), + &ours_root.join(format!("{NAME}_1.0-1_amd64.changes")), + ); + assert_buildinfo_equivalent( + &golden_root.join(format!("{NAME}_1.0-1_amd64.buildinfo")), + &ours_root.join(format!("{NAME}_1.0-1_amd64.buildinfo")), + ); + } + #[test] fn diff_native_minimal() { differential_case(&FixtureSpec::new("pkh-diff-a", "1.0-1", "unstable")); diff --git a/src/deb/local.rs b/src/deb/local.rs index 7af2ced..a9caaf6 100644 --- a/src/deb/local.rs +++ b/src/deb/local.rs @@ -227,6 +227,20 @@ pub async fn build( .to_str() .ok_or("Invalid package directory path")?; + // Reproducibility: export SOURCE_DATE_EPOCH from the changelog entry, + // like dpkg-buildpackage does. + match ctx.read_file(&package_dir.join("debian/changelog")) { + Ok(content) => { + if let Ok(entry) = crate::debian::parse_changelog_entry_from_str(&content) { + env.insert( + "SOURCE_DATE_EPOCH".to_string(), + entry.timestamp.to_string(), + ); + } + } + Err(e) => log::debug!("cannot read changelog for SOURCE_DATE_EPOCH: {}", e), + } + // Apply quilt patches if the package provides a patch series apply_quilt_patches(package_dir_str, &env, ctx.clone(), &ui, &sink)?; @@ -318,6 +332,97 @@ pub async fn build( ); } + // Generate the upload metadata (.buildinfo + .changes) natively, the + // equivalent of dpkg-genbuildinfo -b + dpkg-genchanges -b, consuming + // debian/files produced by the build. Failures are logged but do not + // discard the produced binaries. + if let Err(e) = + generate_upload_metadata(package_dir_str, build_root, arch, cross, &env, &ctx) + { + warn!("failed to generate .buildinfo/.changes: {}", e); + } + + Ok(()) +} + +/// Generate `.buildinfo` and `.changes` for the finished binary build, +/// inside the build context. +fn generate_upload_metadata( + package_dir: &str, + build_root: &str, + arch: &str, + cross: bool, + env: &HashMap, + ctx: &Arc, +) -> Result<(), Box> { + use std::path::Path; + + let changelog_path = Path::new(package_dir).join("debian/changelog"); + let changelog_content = ctx.read_file(&changelog_path)?; + let entry = + crate::debian::parse_changelog_entry_from_str(&changelog_content)?; + + // Build architecture: the machine inside the build context. + let build_arch = ctx + .command("dpkg") + .arg("--print-architecture") + .output() + .ok() + .filter(|o| o.status.success()) + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) + .filter(|s| !s.is_empty()) + .unwrap_or_else(crate::get_current_arch); + let host_arch = if cross { + arch.to_string() + } else { + build_arch.clone() + }; + + // Vendor resolution inside the context (falls back to the host view). + let vendor = ctx + .read_file(Path::new("/etc/dpkg/origins/default")) + .ok() + .and_then(|content| { + for line in content.lines() { + if let Some(v) = line.strip_prefix("Vendor:") { + let v = v.trim(); + if !v.is_empty() { + return Some(v.to_string()); + } + } + } + None + }) + .unwrap_or_else(crate::build::env::current_vendor); + + let profiles = crate::build::env::resolve_build_profiles( + &[], + &vendor, + ); + let source_date_epoch = env + .get("SOURCE_DATE_EPOCH") + .and_then(|v| v.parse::().ok()) + .unwrap_or(entry.timestamp); + + let opts = crate::build::binary::BinaryMetadataOptions { + profiles, + vendor, + parallel: crate::build::env::num_parallel(), + source_date_epoch, + build_arch, + host_arch, + }; + let (buildinfo, changes) = crate::build::binary::generate_binary_metadata( + ctx, + Path::new(package_dir), + Path::new(build_root), + &opts, + )?; + log::info!( + "generated upload metadata: {} and {}", + buildinfo.display(), + changes.display() + ); Ok(()) } diff --git a/src/deb/mod.rs b/src/deb/mod.rs index 7edbbfd..2cf440b 100644 --- a/src/deb/mod.rs +++ b/src/deb/mod.rs @@ -165,14 +165,18 @@ async fn build_binary_package_impl( } } - // Retrieve produced .deb files + // Retrieve produced artifacts (.deb files plus the upload metadata + // (.buildinfo/.changes) generated natively after the build) if let Some(u) = ui { u.phase(Phase::RetrievingArtifacts); } let remote_files = build_ctx.list_files(Path::new(&build_root))?; let deb_files: Vec = remote_files .into_iter() - .filter(|f| f.extension().is_some_and(|ext| ext == "deb")) + .filter(|f| { + f.extension() + .is_some_and(|ext| matches!(ext.to_str(), Some("deb") | Some("buildinfo") | Some("changes"))) + }) .collect(); let total_debs = deb_files.len(); diff --git a/src/debian/changelog.rs b/src/debian/changelog.rs index aaad3f7..c2d9c50 100644 --- a/src/debian/changelog.rs +++ b/src/debian/changelog.rs @@ -46,7 +46,15 @@ pub fn parse_changelog_entry(path: &Path) -> Result Result> { + let origin = "changelog"; let mut lines = content.lines().peekable(); // --- Header line: `package (version) distributions; urgency=medium[, key=value]` @@ -55,17 +63,13 @@ pub fn parse_changelog_entry(path: &Path) -> Result continue, Some(l) => break l.trim_end(), None => { - return Err(format!("changelog '{}' is empty", path.display()).into()); + return Err(format!("changelog '{origin}' is empty").into()); } } }; let open = header.find('(').ok_or_else(|| { - format!( - "invalid changelog header in '{}': {}", - path.display(), - header - ) + format!("invalid changelog header in '{origin}': {header}") })?; let close = header[open..] .find(')') @@ -123,9 +127,8 @@ pub fn parse_changelog_entry(path: &Path) -> Result Date'", - path.display() + "no maintainer trailer found in '{origin}': expected a line of the form \ + ' -- Name Date'" ) })?; @@ -148,12 +151,7 @@ pub fn parse_changelog_entry(path: &Path) -> Result Option { pub fn parse_previous_version(path: &Path) -> Result, Box> { let content = std::fs::read_to_string(path) .map_err(|e| format!("failed to read changelog '{}': {}", path.display(), e))?; + parse_previous_version_from_str(&content) +} +/// Return the version of the *previous* changelog entry from the textual +/// content of a changelog file. +pub fn parse_previous_version_from_str( + content: &str, +) -> Result, Box> { let mut seen_first = false; for line in content.lines() { let line = line.trim_end(); @@ -229,12 +234,12 @@ pub fn parse_previous_version(path: &Path) -> Result, Box Result> { - let mut list = FilesList::new(); let content = match std::fs::read_to_string(path) { Ok(c) => c, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(list), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(FilesList::new()), Err(e) => { return Err(format!("cannot read '{}': {}", path.display(), e).into()); } }; + FilesList::parse(&content) + .map_err(|e| format!("in '{}': {}", path.display(), e).into()) + } + + /// Parse a `debian/files` registry from its textual content + /// (`filename section priority [key=value...]` lines). + pub fn parse(content: &str) -> Result { + let mut list = FilesList::new(); for line in content.lines() { if line.trim().is_empty() { continue; } let tokens: Vec<&str> = line.split_whitespace().collect(); if tokens.len() < 3 { - return Err(format!("badly formed line in '{}': {}", path.display(), line).into()); + return Err(format!("badly formed line: {line}")); } - let mut entry = parse_filename(tokens[0]).ok_or_else(|| { - format!( - "badly formed file name in '{}': {}", - path.display(), - tokens[0] - ) - })?; + let mut entry = parse_filename(tokens[0]) + .ok_or_else(|| format!("badly formed file name: {}", tokens[0]))?; entry.section = tokens[1].to_string(); entry.priority = tokens[2].to_string(); for attr in &tokens[3..] { @@ -181,6 +183,23 @@ impl FilesList { self.files.is_empty() } + /// Render the registry to its textual `debian/files` representation. + pub fn render(&self) -> String { + let mut out = String::new(); + for entry in self.iter() { + out.push_str(&entry.filename); + out.push(' '); + out.push_str(&entry.section); + out.push(' '); + out.push_str(&entry.priority); + for (k, v) in &entry.attrs { + out.push_str(&format!(" {k}={v}")); + } + out.push('\n'); + } + out + } + /// Save atomically: write `.new` then rename over `path`, like /// dpkg does. pub fn save_atomic(&self, path: &Path) -> Result<(), Box> { diff --git a/src/debian/mod.rs b/src/debian/mod.rs index a977f81..c869fa8 100644 --- a/src/debian/mod.rs +++ b/src/debian/mod.rs @@ -19,7 +19,10 @@ pub mod deps; pub mod files; pub mod version; -pub use changelog::{ChangelogEntry, parse_changelog_entry}; +pub use changelog::{ + ChangelogEntry, parse_changelog_entry, parse_changelog_entry_from_str, + parse_previous_version_from_str, +}; pub use checksums::{Entry as ChecksumEntry, FileChecksums}; pub use control::{ControlInfo, Paragraph, parse_paragraphs, write_paragraph}; pub use files::{FilesEntry, FilesList};