vhaudiquet/pkh
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

new: add upstream-aware orig tarball origins and the orig-vendor component

Browse Source
This commit is contained in:
vhaudiquet
2026-09-17 01:31:35 +02:00
parent 8e06b2074d
commit 77420e723a
19 changed files with 3270 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/build/mod.rs
+384 -24
View File
@@ -14,6 +14,7 @@ pub mod env;
use std::collections::{BTreeMap, HashMap};
use std::error::Error;
use std::io::IsTerminal;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::Arc;
@@ -61,6 +62,12 @@ pub struct SourceBuildOutput {
/// bar + rolling pane) and tee'd to a log file; on failure the view prints a
/// summary of the last captured errors. Without a UI, commands inherit the
/// terminal as before.
///
/// A `dpkg-source -b` failure is classified (see
/// [`classify_dpkg_source_failure`]); when the vendored rust dependencies
/// diverged from the orig-vendor component and a terminal is attached, the
/// flow offers to re-vendor, recreate the component and retry the build
/// exactly once.
pub fn build_source_package(
cwd: Option<&Path>,
ui: Option<Arc<DebUi>>,
@@ -68,6 +75,9 @@ pub fn build_source_package(
let cwd = cwd.unwrap_or_else(|| Path::new("."));
let output = match run_source_build(cwd, &SourceBuildOptions::default(), ui.clone()) {
Ok(output) => output,
Err(e) if e.downcast_ref::<VendorDriftError>().is_some() => {
return retry_after_revendor(cwd, ui, e);
}
Err(e) => {
if let Some(u) = &ui {
u.finish_failure();
@@ -106,6 +116,83 @@ pub fn build_source_package(
Ok(())
}
/// The re-vendor retry hook for a [`VendorDriftError`]: on an interactive
/// terminal, offer to re-run the vendoring step (the same helper the rust
/// template uses at scaffold time), recreate the `orig-vendor` component
/// from the fresh `vendor/` tree and retry the source build exactly once.
/// Without a terminal (or on a declined offer) the original error is
/// returned untouched.
fn retry_after_revendor(
cwd: &Path,
ui: Option<Arc<DebUi>>,
original: Box<dyn Error>,
) -> Result<(), Box<dyn Error>> {
let interactive = std::io::stdin().is_terminal() && std::io::stdout().is_terminal();
if !interactive {
if let Some(u) = &ui {
u.finish_failure();
}
return Err(original);
}
log::error!("{original}");
let retry = crate::ui::prompt::confirm(
"Re-vendor the Cargo dependencies and retry the build?",
false,
)
.unwrap_or(false);
if !retry {
if let Some(u) = &ui {
u.finish_failure();
}
return Err(original);
}
// 1. Re-vendor into the tree. The vendoring helper never overwrites an
// existing `.cargo/config.toml` (scaffold-time safety); in a package
// we vendored before, that config is ours — remove it so it is
// refreshed. A foreign config (no vendored-source marker) is left
// alone and makes the vendoring report incomplete below.
let config = cwd.join(".cargo/config.toml");
if config.exists()
&& std::fs::read_to_string(&config)
.is_ok_and(|content| content.contains("[source.crates-io]"))
{
std::fs::remove_file(&config)?;
}
if !crate::new::templates::rust::vendor_dependencies(cwd)? {
if let Some(u) = &ui {
u.finish_failure();
}
return Err(
"Re-vendoring did not complete: the tree is unchanged, fix the \
vendoring by hand and build again."
.into(),
);
}
// 2. Recreate the orig-vendor component from the fresh vendor/ tree,
// under the name dpkg-source globs for: the changelog version's
// UPSTREAM part (`0.14.0`, not the full `0.14.0-1` — the stale
// component would survive and the next build would fail again).
let entry = crate::debian::parse_changelog_entry(&cwd.join("debian/changelog"))?;
let uversion = crate::new::orig::component_upstream_version(&entry.version);
let component = crate::new::orig::vendor_component_path(cwd, &entry.source, uversion)
.ok_or_else(|| {
format!(
"cannot determine the output directory of '{}' ",
cwd.display()
)
})?;
if component.exists() {
std::fs::remove_file(&component)?;
}
crate::new::orig::create_vendor_component(cwd, &entry.source, uversion)?;
// 3. One retry.
run_source_build(cwd, &SourceBuildOptions::default(), ui).map(|_| ())
}
/// Run the full native source-build pipeline in `cwd`.
///
/// Steps (mirroring `dpkg-buildpackage -S -I -i -nc -d`):
@@ -280,13 +367,19 @@ pub fn run_source_build(
Box::new(DpkgSourceClassifier::new()),
);
}
run_command(
if let Err(failure) = run_command_capturing(
cwd,
"dpkg-source",
&["-I", "-i", "-b", "."],
&pipeline_env,
sink.as_ref(),
)?;
) {
return Err(dpkg_source_failure_error(
classify_dpkg_source_failure(&failure.stderr),
&failure.stderr,
failure.error,
));
}
if !dsc_path.exists() {
return Err(format!(
@@ -603,19 +696,116 @@ fn parse_checksum_field(field: &str, value: &str) -> Result<Vec<ChecksumLine>, S
Ok(entries)
}
/// Why a `dpkg-source -b` run failed, classified from its captured stderr.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum DpkgSourceFailure {
/// A path under `vendor/` (or the `orig-vendor` component itself) is
/// named in the error: the vendored dependencies changed since the
/// orig-vendor component was created.
VendorDrift,
/// Unrepresentable changes to source outside `vendor/`: the upstream
/// tree drifted beyond the orig snapshot.
UpstreamDrift,
/// Anything else (changelog errors, missing files, …).
Other,
}
/// Classify a captured `dpkg-source -b` stderr. Any error line naming a
/// `vendor/` path or the `orig-vendor` component wins (the vendored tree is
/// what the build choked on); the bare "unrepresentable changes to source"
/// summary without a vendor mention points at general upstream drift.
pub(crate) fn classify_dpkg_source_failure(stderr: &str) -> DpkgSourceFailure {
if stderr
.lines()
.any(|line| line.contains("vendor/") || line.contains("orig-vendor"))
{
return DpkgSourceFailure::VendorDrift;
}
if stderr.contains("unrepresentable changes to source") {
return DpkgSourceFailure::UpstreamDrift;
}
DpkgSourceFailure::Other
}
/// The typed error of the [`DpkgSourceFailure::VendorDrift`] case, letting
/// the build wrapper offer the re-vendor retry. The type must survive all
/// the way to [`build_source_package`], so the offending dpkg-source line
/// travels inside the error instead of being string-wrapped around it.
#[derive(Debug)]
pub(crate) struct VendorDriftError {
/// The offending dpkg-source stderr line, when one was captured.
detail: Option<String>,
}
impl std::fmt::Display for VendorDriftError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"dpkg-source failed: the vendored dependencies changed since \
the orig-vendor component was created. Re-vendor the tree \
(`cargo vendor`) and recreate the <name>_<version>.\
orig-vendor.tar.xz component, then build again."
)?;
if let Some(detail) = &self.detail {
write!(f, " (dpkg-source said: {detail})")?;
}
Ok(())
}
}
impl Error for VendorDriftError {}
/// The user-facing error for a classified `dpkg-source -b` failure: a
/// pointed explanation for the two known drift cases, the raw command
/// failure otherwise.
fn dpkg_source_failure_error(
failure: DpkgSourceFailure,
stderr: &str,
command_error: Box<dyn Error>,
) -> Box<dyn Error> {
match failure {
DpkgSourceFailure::VendorDrift => {
let detail = stderr
.lines()
.find(|line| line.contains("vendor"))
.map(str::trim)
.filter(|line| !line.is_empty());
Box::new(VendorDriftError {
detail: detail.map(str::to_string),
})
}
DpkgSourceFailure::UpstreamDrift => format!(
"dpkg-source failed: the upstream tree drifted beyond the orig \
snapshot — bump the version (`pkh chlog`) or re-scaffold. \
(raw error: {command_error})"
)
.into(),
DpkgSourceFailure::Other => command_error,
}
}
/// A command failure together with the stderr captured while it ran (empty
/// when the streams were inherited, e.g. verbose mode).
struct CommandFailure {
error: Box<dyn Error>,
stderr: String,
}
/// Run a build command in `cwd` with extra environment variables layered on
/// top of the inherited environment.
///
/// When `sink` is set, stdout/stderr are piped and every line is forwarded to
/// it (live view + tee log); otherwise stdio is inherited from the terminal.
/// Returns an error on non-zero exit status.
fn run_command(
/// it (live view + tee log) while the stderr is additionally captured for
/// the failure classification; otherwise stdio is inherited from the
/// terminal. Returns an error (with the captured stderr) on non-zero exit
/// status.
fn run_command_capturing(
cwd: &Path,
program: &str,
args: &[&str],
env: &BTreeMap<String, String>,
sink: Option<&Arc<dyn LineSink>>,
) -> Result<(), Box<dyn Error>> {
) -> Result<(), CommandFailure> {
log::debug!(
"running: {} {} (in {})",
program,
@@ -626,15 +816,21 @@ fn run_command(
let mut cmd = Command::new(program);
cmd.current_dir(cwd).envs(env).args(args);
// The last 64 KiB of stderr, kept for the dpkg-source failure
// classification.
let stderr_capture = Arc::new(std::sync::Mutex::new(String::new()));
let status = match sink {
None => cmd
.status()
.map_err(|e| format!("failed to run '{}': {}", program, e))?,
None => cmd.status().map_err(|e| CommandFailure {
error: format!("failed to run '{}': {}", program, e).into(),
stderr: String::new(),
})?,
Some(sink) => {
cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
let mut child = cmd
.spawn()
.map_err(|e| format!("failed to run '{}': {}", program, e))?;
let mut child = cmd.spawn().map_err(|e| CommandFailure {
error: format!("failed to run '{}': {}", program, e).into(),
stderr: String::new(),
})?;
let stdout = child.stdout.take();
let stderr = child.stderr.take();
@@ -642,37 +838,89 @@ fn run_command(
// approximate (channel arrival order), acceptable for display.
let out_sink = sink.clone();
let err_sink = sink.clone();
let err_capture = Arc::clone(&stderr_capture);
let out_thread = std::thread::spawn(move || {
if let Some(out) = stdout {
pump(out, Stream::Stdout, &*out_sink);
}
});
let err_thread = std::thread::spawn(move || {
if let Some(err) = stderr {
pump(err, Stream::Stderr, &*err_sink);
if let Some(stderr) = stderr {
pump(
stderr,
Stream::Stderr,
&CapturingSink {
inner: err_sink,
capture: err_capture,
},
);
}
});
let _ = out_thread.join();
let _ = err_thread.join();
child
.wait()
.map_err(|e| format!("failed to wait for '{}': {}", program, e))?
child.wait().map_err(|e| CommandFailure {
error: format!("failed to wait for '{}': {}", program, e).into(),
stderr: String::new(),
})?
}
};
if !status.success() {
return Err(format!(
"'{} {}' failed with status: {}",
program,
args.join(" "),
status
)
.into());
return Err(CommandFailure {
error: format!(
"'{} {}' failed with status: {}",
program,
args.join(" "),
status
)
.into(),
stderr: std::mem::take(&mut stderr_capture.lock().unwrap_or_else(|e| e.into_inner())),
});
}
Ok(())
}
/// Run a build command, discarding the captured stderr.
fn run_command(
cwd: &Path,
program: &str,
args: &[&str],
env: &BTreeMap<String, String>,
sink: Option<&Arc<dyn LineSink>>,
) -> Result<(), Box<dyn Error>> {
run_command_capturing(cwd, program, args, env, sink).map_err(|failure| failure.error)
}
/// A [`LineSink`] forwarding every line to the live view while appending
/// stderr lines to the classification buffer.
struct CapturingSink {
inner: Arc<dyn LineSink>,
capture: Arc<std::sync::Mutex<String>>,
}
const STDERR_CAPTURE_LIMIT: usize = 64 * 1024;
impl LineSink for CapturingSink {
fn line(&self, stream: Stream, line: &str) {
if stream == Stream::Stderr {
let mut buffer = self.capture.lock().unwrap_or_else(|e| e.into_inner());
buffer.push_str(line);
buffer.push('\n');
// Keep the tail: dpkg-source reports the offending path last.
if buffer.len() > STDERR_CAPTURE_LIMIT {
let cut = buffer.len() - STDERR_CAPTURE_LIMIT;
let start = buffer[cut..]
.find('\n')
.map(|index| cut + index + 1)
.unwrap_or(buffer.len());
buffer.drain(..start);
}
}
self.inner.line(stream, line);
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -757,6 +1005,118 @@ mod tests {
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].name, "hello.tar.xz");
}
/// Real-shape `dpkg-source -b` stderr excerpts and their class.
#[test]
fn dpkg_source_failure_classification() {
// Vendored dependencies diverged from the orig-vendor component
// (dpkg names the offending paths before the summary line).
let vendor = concat!(
"dpkg-source: info: building mytool using existing \
./mytool_0.1.0.orig.tar.xz\n",
"dpkg-source: info: using source format version 3.0 (quilt)\n",
"dpkg-source: error: cannot represent change to \
vendor/libc/src/unix/linux_like/mod.rs: binary file contents \
changed\n",
"dpkg-source: error: unrepresentable changes to source\n",
);
assert_eq!(
classify_dpkg_source_failure(vendor),
DpkgSourceFailure::VendorDrift
);
// A missing orig-vendor component makes the vendored files look
// "new"; still vendor drift.
let missing_component = concat!(
"dpkg-source: error: cannot represent change to \
vendor/libc/Cargo.toml: new file is binary\n",
"dpkg-source: error: unrepresentable changes to source\n",
);
assert_eq!(
classify_dpkg_source_failure(missing_component),
DpkgSourceFailure::VendorDrift
);
// The component named in the error is vendor drift too.
assert_eq!(
classify_dpkg_source_failure(
"dpkg-source: error: orig-vendor component tarball \
checksum mismatch\n",
),
DpkgSourceFailure::VendorDrift
);
// Upstream tree drift outside vendor/.
let upstream = concat!(
"dpkg-source: error: cannot represent change to \
assets/logo.png: binary file contents changed\n",
"dpkg-source: error: unrepresentable changes to source\n",
);
assert_eq!(
classify_dpkg_source_failure(upstream),
DpkgSourceFailure::UpstreamDrift
);
assert_eq!(
classify_dpkg_source_failure("dpkg-source: error: unrepresentable changes to source\n"),
DpkgSourceFailure::UpstreamDrift
);
// Anything else.
assert_eq!(
classify_dpkg_source_failure(
"dpkg-source: error: syntax error in debian/control at line 3\n"
),
DpkgSourceFailure::Other
);
assert_eq!(classify_dpkg_source_failure(""), DpkgSourceFailure::Other);
}
/// The user-facing wording of the classified failures.
#[test]
fn dpkg_source_failure_messages() {
let vendor = "dpkg-source: error: cannot represent change to \
vendor/serde/src/de/mod.rs: binary file contents changed\n\
dpkg-source: error: unrepresentable changes to source\n";
let error = dpkg_source_failure_error(
DpkgSourceFailure::VendorDrift,
vendor,
"'dpkg-source -I -i -b .' failed with status: exit status: 2".into(),
);
let message = error.to_string();
assert!(
message.contains("vendored dependencies changed"),
"{message}"
);
assert!(message.contains("orig-vendor"), "{message}");
// The offending dpkg line travels along as the detail.
assert!(message.contains("vendor/serde/src/de/mod.rs"), "{message}");
// Regression: the typed error must SURVIVE the message building, or
// the re-vendor retry hook never fires.
assert!(error.downcast_ref::<VendorDriftError>().is_some());
let upstream_error = dpkg_source_failure_error(
DpkgSourceFailure::UpstreamDrift,
"dpkg-source: error: unrepresentable changes to source\n",
"'dpkg-source -I -i -b .' failed with status: exit status: 2".into(),
);
let message = upstream_error.to_string();
assert!(
message.contains("drifted beyond the orig snapshot"),
"{message}"
);
assert!(message.contains("pkh chlog"), "{message}");
// Other: the raw command error passes through untouched.
let other = dpkg_source_failure_error(DpkgSourceFailure::Other, "", "boom".into());
assert_eq!(other.to_string(), "boom");
// A vendor-drift error (with or without detail) is recognized by
// the retry hook.
for detail in [None, Some("dpkg-source: error: nope".to_string())] {
let boxed: Box<dyn Error> = Box::new(VendorDriftError { detail });
assert!(boxed.downcast_ref::<VendorDriftError>().is_some());
}
}
}
/// Differential tests: build synthetic (or real archive) source packages