vhaudiquet/pkh
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

build/binary: native .buildinfo/.changes for binary builds (pkh deb)

Browse Source 
Extend the metadata writers to binary-only uploads and wire them into
the 'pkh deb' flow:

- build/binary.rs generates <pkg>_<ver>_<arch>.buildinfo/.changes
  through any Context: debian/files consumption, encounter-order
  Architecture accumulation (sorted in .buildinfo like dpkg-genbuildinfo),
  sorted Binary lists, dpkg-formatted Description lines with udeb
  suffixes, Installed-Build-Depends closure over the context status DB,
  and binNMU handling (Source: pkg (prev), Binary-Only-Changes, previous
  .dsc redistribution);
- artifact digests are computed inside the context via coreutils
  (md5sum/sha1sum/sha256sum/stat) so chrooted/remote trees work;
- deb/local.rs runs the generation after 'rules binary', exports
  SOURCE_DATE_EPOCH from the changelog (reproducibility), and resolves
  vendor/profiles inside the context; deb/mod.rs retrieves the new
  artifacts alongside the debs;
- reusable helpers added: FilesList::parse/render,
  parse_changelog_entry_from_str, parse_previous_version_from_str,
  installed_build_depends_from_content.

Differential gate: same tree built with real 'dpkg-buildpackage -b' and
with the pkh flow; .changes/.buildinfo compared field-by-field modulo
machine-dependent fields, artifact checksums included.
This commit is contained in:
vhaudiquet
2026-08-24 11:58:19 +02:00
parent dfaab0606a
commit 42fcfc2dfa
9 changed files with 808 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/build/binary.rs
+493
View File
@@ -0,0 +1,493 @@
//! Binary-build metadata generation: native `.buildinfo` / `.changes`
//! production for binary-only builds (`pkh deb`), the equivalent of
//! `dpkg-genbuildinfo -b` + `dpkg-genchanges -b`.
//!
//! All tree/database access goes through a [`Context`] so the generation can
//! run against a build tree living in a local directory, an ephemeral
//! chroot or a remote host. Artifact digests are computed inside the context
//! with coreutils (`md5sum`, `sha1sum`, `sha256sum`, `stat`), keeping the
//! flow binary-safe regardless of the transport.
use std::collections::BTreeMap;
use std::error::Error;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::context::Context;
use crate::debian::{
parse_changelog_entry_from_str, ChecksumEntry, ControlInfo, FileChecksums, FilesList,
};
/// Digests of one artifact.
#[derive(Debug, Clone, Default)]
struct ArtifactHashes {
size: u64,
md5: String,
sha1: String,
sha256: String,
}
/// Options driving binary metadata generation.
#[derive(Debug, Clone)]
pub struct BinaryMetadataOptions {
/// Active build profiles (`Built-For-Profiles`).
pub profiles: Vec<String>,
/// Vendor name (`Build-Origin`).
pub vendor: String,
/// Parallel job count advertised in `DEB_BUILD_OPTIONS`.
pub parallel: usize,
/// Reproducible-builds epoch exported to the build.
pub source_date_epoch: i64,
/// Build architecture (the machine inside the build context).
pub build_arch: String,
/// Host architecture (the packages' target); equals the build
/// architecture except for cross builds.
pub host_arch: String,
}
/// Generate `<pkg>_<ver>_<arch>.buildinfo` and `.changes` for a finished
/// binary build, consuming `debian/files` from `package_dir` and the
/// artifacts sitting in `upload_dir`. Returns both paths (inside the
/// context).
///
/// Mirrors the observable behavior of `dpkg-genbuildinfo -b` and
/// `dpkg-genchanges -b`: sorted `Binary` list, encounter-order `Architecture`
/// accumulation, sorted `Description` lines formatted like dpkg, `.buildinfo`
/// registration in `debian/files`, and binary-NMU handling (`Source:
/// pkg (prev)` + previous `.dsc` redistribution when present).
pub fn generate_binary_metadata(
ctx: &Arc<Context>,
package_dir: &Path,
upload_dir: &Path,
opts: &BinaryMetadataOptions,
) -> Result<(PathBuf, PathBuf), Box<dyn Error>> {
// ------------------------------------------------------------------
// Metadata sources inside the context
// ------------------------------------------------------------------
let changelog_content = ctx.read_file(&package_dir.join("debian/changelog"))?;
let entry = parse_changelog_entry_from_str(&changelog_content)?;
let control_content = ctx.read_file(&package_dir.join("debian/control"))?;
let control = ControlInfo::parse_content(&control_content)?;
let files_content = ctx
.read_file(&package_dir.join("debian/files"))
.unwrap_or_default();
let mut files_list = FilesList::parse(&files_content)?;
// ------------------------------------------------------------------
// Collect binary artifacts registered in debian/files
// ------------------------------------------------------------------
let artifact_names: Vec<String> = files_list
.iter()
.filter(|e| {
matches!(e.package_type.as_deref(), Some("deb") | Some("udeb"))
})
.map(|e| e.filename.clone())
.collect();
if artifact_names.is_empty() {
return Err(
"binary build with no binary artifacts found; cannot distribute".into(),
);
}
let mut hashes = hashes_in_context(ctx, upload_dir, &artifact_names)?;
let mut checksums = FileChecksums::new();
let mut arch_values: Vec<String> = Vec::new();
let mut arch_seen = std::collections::HashSet::new();
for name in &artifact_names {
let entry_hashes = hashes.remove(name).ok_or_else(|| {
format!("artifact '{name}' listed in debian/files but not found")
})?;
checksums.insert_entry(
name,
ChecksumEntry {
size: entry_hashes.size,
md5: entry_hashes.md5,
sha1: entry_hashes.sha1,
sha256: entry_hashes.sha256,
},
);
// Architecture accumulation in encounter order (dpkg-genchanges).
if let Some(file_entry) = files_list.get(name)
&& let Some(arch) = file_entry.arch.as_ref().or_else(|| {
file_entry.attrs.get("architecture")
})
&& arch_seen.insert(arch.clone())
{
arch_values.push(arch.clone());
}
}
// ------------------------------------------------------------------
// Binary-NMU: redistribute the previous source when present
// ------------------------------------------------------------------
let sversion = entry.version.no_epoch();
let mut source_display = entry.source.clone();
let mut binary_only_changes = None;
if entry.binary_only
&& let Ok(prev_entry) =
crate::debian::changelog::parse_previous_version_from_str(&ctx.read_file(
&package_dir.join("debian/changelog"),
)?)
&& let Some(prev) = prev_entry
{
source_display = format!("{} ({})", entry.source, prev);
binary_only_changes = Some(format!(
"{}\n\n -- {} <{}> {}",
entry.changes_field,
entry.maintainer_name,
entry.maintainer_email,
entry.date_raw
));
let prev_version = crate::debian::DebianVersion::parse(&prev)?;
let dsc_name = format!("{}_{}.dsc", entry.source, prev_version.no_epoch());
let dsc_path = upload_dir.join(&dsc_name);
if ctx.exists(&dsc_path)? {
include_dsc_artifacts(ctx, upload_dir, &dsc_name, &mut checksums)?;
}
}
// ------------------------------------------------------------------
// Binary package names and descriptions
// ------------------------------------------------------------------
let mut binaries: Vec<String> = Vec::new();
for name in &artifact_names {
if let Some(e) = files_list.get(name)
&& let Some(pkg) = &e.package
&& !binaries.contains(pkg)
{
binaries.push(pkg.clone());
}
}
binaries.sort();
// Description lines: first line of each binary stanza's Description,
// formatted exactly like dpkg-genchanges, sorted.
let mut descriptions = Vec::new();
for stanza in &control.binaries {
let Some(pkg) = stanza.get("Package") else {
continue;
};
if !binaries.contains(&pkg.to_string()) {
continue;
}
let summary = stanza
.get("Description")
.unwrap_or("no description available")
.lines()
.next()
.unwrap_or("no description available");
// Package-Type overrides the artifact-derived type (deb default).
let pkg_type = stanza
.get("Package-Type")
.map(str::to_string)
.unwrap_or_else(|| {
files_list
.iter()
.find(|f| f.package.as_deref() == Some(pkg))
.and_then(|f| f.package_type.clone())
.unwrap_or_else(|| "deb".to_string())
});
descriptions.push(crate::build::changes::format_description(
pkg, &pkg_type, summary,
));
}
descriptions.sort();
// ------------------------------------------------------------------
// Installed-Build-Depends closure over the context status database
// ------------------------------------------------------------------
let status_content = ctx
.read_file(Path::new("/var/lib/dpkg/status"))
.unwrap_or_default();
let bd_fields = [
control.source.get("Build-Depends").unwrap_or(""),
control.source.get("Build-Depends-Arch").unwrap_or(""),
control.source.get("Build-Depends-Indep").unwrap_or(""),
];
let installed_build_depends =
crate::build::buildinfo::installed_build_depends_from_content(
&status_content,
&bd_fields,
)?;
// ------------------------------------------------------------------
// .buildinfo generation, then registration in debian/files
// ------------------------------------------------------------------
let pipeline_env = pipeline_environment(opts);
let environment = crate::build::env::buildinfo_environment(&pipeline_env);
// dpkg-genbuildinfo sorts the accumulated architecture values, while
// dpkg-genchanges keeps encounter order.
let mut buildinfo_arch_values = arch_values.clone();
buildinfo_arch_values.sort();
let buildinfo_name = format!(
"{}_{}_{}.buildinfo",
entry.source,
sversion,
opts.host_arch
);
let buildinfo_doc = crate::build::buildinfo::render_buildinfo(
&crate::build::buildinfo::BuildInfoInput {
source: source_display.clone(),
binaries: binaries.clone(),
architecture: buildinfo_arch_values.join(" "),
version: entry.version.full(),
binary_only_changes: binary_only_changes.clone(),
build_origin: opts.vendor.clone(),
build_architecture: opts.build_arch.clone(),
build_date: chrono::Local::now().to_rfc2822(),
checksums: checksums.clone(),
installed_build_depends,
environment,
},
);
let buildinfo_path = upload_dir.join(&buildinfo_name);
ctx.write_file(&buildinfo_path, &crate::debian::control::write_paragraph(&buildinfo_doc))?;
// Register the .buildinfo in debian/files, like dpkg-genbuildinfo does,
// so the .changes distributes it.
files_list.add(crate::debian::FilesEntry::new(
&buildinfo_name,
control.section(),
control.priority(),
));
ctx.write_file(
&package_dir.join("debian/files"),
&files_list.render(),
)?;
// Hash the freshly written .buildinfo inside the context.
let buildinfo_hashes = hashes_in_context(
ctx,
upload_dir,
std::slice::from_ref(&buildinfo_name),
)?;
if let Some(h) = buildinfo_hashes.get(&buildinfo_name) {
checksums.insert_entry(
&buildinfo_name,
ChecksumEntry {
size: h.size,
md5: h.md5.clone(),
sha1: h.sha1.clone(),
sha256: h.sha256.clone(),
},
);
}
// ------------------------------------------------------------------
// .changes generation
// ------------------------------------------------------------------
let changes_name = format!("{}_{}_{}.changes", entry.source, sversion, opts.host_arch);
let changed_by = format!("{} <{}>", entry.maintainer_name, entry.maintainer_email);
let changes_doc = crate::build::changes::render_changes(&crate::build::changes::ChangesInput {
date: entry.date_raw.clone(),
source: source_display,
binaries,
built_for_profiles: opts.profiles.clone(),
architecture: arch_values.join(" "),
version: entry.version.full(),
distribution: entry.distribution.clone(),
urgency: entry.urgency.clone(),
maintainer: control.source.get("Maintainer").map(str::to_string),
changed_by: Some(changed_by),
descriptions,
closes: entry.closes.clone(),
changes_field: entry.changes_field.clone(),
checksums,
files_list,
});
let changes_path = upload_dir.join(&changes_name);
ctx.write_file(&changes_path, &crate::debian::control::write_paragraph(&changes_doc))?;
Ok((buildinfo_path, changes_path))
}
/// Environment exported to the build steps; recorded (filtered) in the
/// `.buildinfo` `Environment` field.
fn pipeline_environment(opts: &BinaryMetadataOptions) -> BTreeMap<String, String> {
let mut env = BTreeMap::new();
env.insert(
"SOURCE_DATE_EPOCH".to_string(),
opts.source_date_epoch.to_string(),
);
env.insert(
"DEB_BUILD_OPTIONS".to_string(),
format!("parallel={}", opts.parallel),
);
if !opts.profiles.is_empty() {
env.insert(
"DEB_BUILD_PROFILES".to_string(),
opts.profiles.join(","),
);
}
env
}
/// Compute md5/sha1/sha256 digests and sizes for the named files inside the
/// context directory `dir`, using coreutils.
fn hashes_in_context(
ctx: &Arc<Context>,
dir: &Path,
names: &[String],
) -> Result<BTreeMap<String, ArtifactHashes>, Box<dyn Error>> {
let mut out: BTreeMap<String, ArtifactHashes> = names
.iter()
.map(|n| (n.clone(), ArtifactHashes::default()))
.collect();
// Sizes.
let output = ctx
.command("stat")
.current_dir(dir)
.arg("-c")
.arg("%s %n")
.args(names)
.output()?;
let stdout = String::from_utf8_lossy(&output.stdout);
for line in stdout.lines() {
let Some((size, name)) = line.trim().split_once(' ') else {
continue;
};
if let Some(slot) = out.get_mut(name) {
slot.size = size.parse().unwrap_or(0);
}
}
// Digests.
for (tool, field) in [
("md5sum", 0usize),
("sha1sum", 1usize),
("sha256sum", 2usize),
] {
let output = ctx
.command(tool)
.current_dir(dir)
.args(names)
.output()
.map_err(|e| format!("failed to run '{tool}' inside the build context: {e}"))?;
if !output.status.success() {
return Err(format!(
"'{tool}' failed inside the build context: {}",
String::from_utf8_lossy(&output.stderr).trim()
)
.into());
}
let stdout = String::from_utf8_lossy(&output.stdout);
for line in stdout.lines() {
let Some((digest, name)) = line.trim().split_once(" ") else {
continue;
};
let name = name.trim_start_matches('*');
if let Some(slot) = out.get_mut(name) {
match field {
0 => slot.md5 = digest.to_string(),
1 => slot.sha1 = digest.to_string(),
_ => slot.sha256 = digest.to_string(),
}
}
}
}
Ok(out)
}
/// Pull the `.dsc` checksums (and its referenced tarballs) into the
/// checksum registry, mirroring how binary-NMU uploads redistribute the
/// previous source.
fn include_dsc_artifacts(
ctx: &Arc<Context>,
upload_dir: &Path,
dsc_name: &str,
checksums: &mut FileChecksums,
) -> Result<(), Box<dyn Error>> {
let dsc_content = ctx.read_file(&upload_dir.join(dsc_name))?;
let para = crate::debian::control::parse_paragraphs(&dsc_content)
.into_iter()
.next()
.ok_or_else(|| format!("'{dsc_name}' is empty"))?;
let mut names: Vec<String> = Vec::new();
let mut partials: BTreeMap<String, PartialDscChecksums> = BTreeMap::new();
for field in ["Checksums-Sha1", "Checksums-Sha256"] {
if let Some(value) = para.get(field) {
for line in value.lines() {
let tokens: Vec<&str> = line.split_whitespace().collect();
if tokens.len() != 3 {
continue;
}
let slot = partials.entry(tokens[2].to_string()).or_default();
if field == "Checksums-Sha1" {
slot.sha1 = Some(tokens[0].to_string());
} else {
slot.sha256 = Some(tokens[0].to_string());
}
slot.size = tokens[1].parse().ok().or(slot.size);
}
}
}
if let Some(files_value) = para.get("Files") {
for line in files_value.lines() {
let tokens: Vec<&str> = line.split_whitespace().collect();
if tokens.len() >= 3 {
let slot = partials.entry(tokens[2].to_string()).or_default();
slot.md5 = Some(tokens[0].to_string());
slot.size = tokens[1].parse().ok().or(slot.size);
}
}
}
for field in ["Checksums-Sha1", "Checksums-Sha256"] {
if let Some(value) = para.get(field) {
for line in value.lines() {
if let Some(name) = line.split_whitespace().nth(2) {
names.push(name.to_string());
}
}
}
}
// The .dsc itself is hashed fresh (it may be signed/rewritten); the
// tarballs reuse the .dsc-recorded digests, like dpkg-genchanges does.
let dsc_hashes = hashes_in_context(ctx, upload_dir, std::slice::from_ref(&dsc_name.to_string()))?;
if let Some(h) = dsc_hashes.get(dsc_name) {
checksums.insert_entry(
dsc_name,
ChecksumEntry {
size: h.size,
md5: h.md5.clone(),
sha1: h.sha1.clone(),
sha256: h.sha256.clone(),
},
);
}
for name in &names {
if name == dsc_name {
continue;
}
let p = &partials[name];
checksums.insert_entry(
name,
ChecksumEntry {
size: p.size.unwrap_or(0),
md5: p.md5.clone().unwrap_or_default(),
sha1: p.sha1.clone().unwrap_or_default(),
sha256: p.sha256.clone().unwrap_or_default(),
},
);
}
Ok(())
}
/// Partially-known checksums taken from a `.dsc` checksum field.
#[derive(Debug, Default)]
struct PartialDscChecksums {
size: Option<u64>,
md5: Option<String>,
sha1: Option<String>,
sha256: Option<String>,
}