diff --git a/src/build/mod.rs b/src/build/mod.rs index ef80f8c..7a7918e 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -14,6 +14,7 @@ pub mod env; use std::collections::{BTreeMap, HashMap}; use std::error::Error; +use std::io::IsTerminal; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::Arc; @@ -61,6 +62,12 @@ pub struct SourceBuildOutput { /// bar + rolling pane) and tee'd to a log file; on failure the view prints a /// summary of the last captured errors. Without a UI, commands inherit the /// terminal as before. +/// +/// A `dpkg-source -b` failure is classified (see +/// [`classify_dpkg_source_failure`]); when the vendored rust dependencies +/// diverged from the orig-vendor component and a terminal is attached, the +/// flow offers to re-vendor, recreate the component and retry the build +/// exactly once. pub fn build_source_package( cwd: Option<&Path>, ui: Option>, @@ -68,6 +75,9 @@ pub fn build_source_package( let cwd = cwd.unwrap_or_else(|| Path::new(".")); let output = match run_source_build(cwd, &SourceBuildOptions::default(), ui.clone()) { Ok(output) => output, + Err(e) if e.downcast_ref::().is_some() => { + return retry_after_revendor(cwd, ui, e); + } Err(e) => { if let Some(u) = &ui { u.finish_failure(); @@ -106,6 +116,83 @@ pub fn build_source_package( Ok(()) } +/// The re-vendor retry hook for a [`VendorDriftError`]: on an interactive +/// terminal, offer to re-run the vendoring step (the same helper the rust +/// template uses at scaffold time), recreate the `orig-vendor` component +/// from the fresh `vendor/` tree and retry the source build exactly once. +/// Without a terminal (or on a declined offer) the original error is +/// returned untouched. +fn retry_after_revendor( + cwd: &Path, + ui: Option>, + original: Box, +) -> Result<(), Box> { + let interactive = std::io::stdin().is_terminal() && std::io::stdout().is_terminal(); + if !interactive { + if let Some(u) = &ui { + u.finish_failure(); + } + return Err(original); + } + + log::error!("{original}"); + let retry = crate::ui::prompt::confirm( + "Re-vendor the Cargo dependencies and retry the build?", + false, + ) + .unwrap_or(false); + if !retry { + if let Some(u) = &ui { + u.finish_failure(); + } + return Err(original); + } + + // 1. Re-vendor into the tree. The vendoring helper never overwrites an + // existing `.cargo/config.toml` (scaffold-time safety); in a package + // we vendored before, that config is ours — remove it so it is + // refreshed. A foreign config (no vendored-source marker) is left + // alone and makes the vendoring report incomplete below. + let config = cwd.join(".cargo/config.toml"); + if config.exists() + && std::fs::read_to_string(&config) + .is_ok_and(|content| content.contains("[source.crates-io]")) + { + std::fs::remove_file(&config)?; + } + if !crate::new::templates::rust::vendor_dependencies(cwd)? { + if let Some(u) = &ui { + u.finish_failure(); + } + return Err( + "Re-vendoring did not complete: the tree is unchanged, fix the \ + vendoring by hand and build again." + .into(), + ); + } + + // 2. Recreate the orig-vendor component from the fresh vendor/ tree, + // under the name dpkg-source globs for: the changelog version's + // UPSTREAM part (`0.14.0`, not the full `0.14.0-1` — the stale + // component would survive and the next build would fail again). + let entry = crate::debian::parse_changelog_entry(&cwd.join("debian/changelog"))?; + let uversion = crate::new::orig::component_upstream_version(&entry.version); + let component = crate::new::orig::vendor_component_path(cwd, &entry.source, uversion) + .ok_or_else(|| { + format!( + "cannot determine the output directory of '{}' ", + cwd.display() + ) + })?; + if component.exists() { + std::fs::remove_file(&component)?; + } + crate::new::orig::create_vendor_component(cwd, &entry.source, uversion)?; + + // 3. One retry. + run_source_build(cwd, &SourceBuildOptions::default(), ui).map(|_| ()) +} + /// Run the full native source-build pipeline in `cwd`. /// /// Steps (mirroring `dpkg-buildpackage -S -I -i -nc -d`): @@ -280,13 +367,19 @@ pub fn run_source_build( Box::new(DpkgSourceClassifier::new()), ); } - run_command( + if let Err(failure) = run_command_capturing( cwd, "dpkg-source", &["-I", "-i", "-b", "."], &pipeline_env, sink.as_ref(), - )?; + ) { + return Err(dpkg_source_failure_error( + classify_dpkg_source_failure(&failure.stderr), + &failure.stderr, + failure.error, + )); + } if !dsc_path.exists() { return Err(format!( @@ -603,19 +696,116 @@ fn parse_checksum_field(field: &str, value: &str) -> Result, S Ok(entries) } +/// Why a `dpkg-source -b` run failed, classified from its captured stderr. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DpkgSourceFailure { + /// A path under `vendor/` (or the `orig-vendor` component itself) is + /// named in the error: the vendored dependencies changed since the + /// orig-vendor component was created. + VendorDrift, + /// Unrepresentable changes to source outside `vendor/`: the upstream + /// tree drifted beyond the orig snapshot. + UpstreamDrift, + /// Anything else (changelog errors, missing files, …). + Other, +} + +/// Classify a captured `dpkg-source -b` stderr. Any error line naming a +/// `vendor/` path or the `orig-vendor` component wins (the vendored tree is +/// what the build choked on); the bare "unrepresentable changes to source" +/// summary without a vendor mention points at general upstream drift. +pub(crate) fn classify_dpkg_source_failure(stderr: &str) -> DpkgSourceFailure { + if stderr + .lines() + .any(|line| line.contains("vendor/") || line.contains("orig-vendor")) + { + return DpkgSourceFailure::VendorDrift; + } + if stderr.contains("unrepresentable changes to source") { + return DpkgSourceFailure::UpstreamDrift; + } + DpkgSourceFailure::Other +} + +/// The typed error of the [`DpkgSourceFailure::VendorDrift`] case, letting +/// the build wrapper offer the re-vendor retry. The type must survive all +/// the way to [`build_source_package`], so the offending dpkg-source line +/// travels inside the error instead of being string-wrapped around it. +#[derive(Debug)] +pub(crate) struct VendorDriftError { + /// The offending dpkg-source stderr line, when one was captured. + detail: Option, +} + +impl std::fmt::Display for VendorDriftError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "dpkg-source failed: the vendored dependencies changed since \ + the orig-vendor component was created. Re-vendor the tree \ + (`cargo vendor`) and recreate the _.\ + orig-vendor.tar.xz component, then build again." + )?; + if let Some(detail) = &self.detail { + write!(f, " (dpkg-source said: {detail})")?; + } + Ok(()) + } +} + +impl Error for VendorDriftError {} + +/// The user-facing error for a classified `dpkg-source -b` failure: a +/// pointed explanation for the two known drift cases, the raw command +/// failure otherwise. +fn dpkg_source_failure_error( + failure: DpkgSourceFailure, + stderr: &str, + command_error: Box, +) -> Box { + match failure { + DpkgSourceFailure::VendorDrift => { + let detail = stderr + .lines() + .find(|line| line.contains("vendor")) + .map(str::trim) + .filter(|line| !line.is_empty()); + Box::new(VendorDriftError { + detail: detail.map(str::to_string), + }) + } + DpkgSourceFailure::UpstreamDrift => format!( + "dpkg-source failed: the upstream tree drifted beyond the orig \ + snapshot — bump the version (`pkh chlog`) or re-scaffold. \ + (raw error: {command_error})" + ) + .into(), + DpkgSourceFailure::Other => command_error, + } +} + +/// A command failure together with the stderr captured while it ran (empty +/// when the streams were inherited, e.g. verbose mode). +struct CommandFailure { + error: Box, + stderr: String, +} + /// Run a build command in `cwd` with extra environment variables layered on /// top of the inherited environment. /// /// When `sink` is set, stdout/stderr are piped and every line is forwarded to -/// it (live view + tee log); otherwise stdio is inherited from the terminal. -/// Returns an error on non-zero exit status. -fn run_command( +/// it (live view + tee log) while the stderr is additionally captured for +/// the failure classification; otherwise stdio is inherited from the +/// terminal. Returns an error (with the captured stderr) on non-zero exit +/// status. +fn run_command_capturing( cwd: &Path, program: &str, args: &[&str], env: &BTreeMap, sink: Option<&Arc>, -) -> Result<(), Box> { +) -> Result<(), CommandFailure> { log::debug!( "running: {} {} (in {})", program, @@ -626,15 +816,21 @@ fn run_command( let mut cmd = Command::new(program); cmd.current_dir(cwd).envs(env).args(args); + // The last 64 KiB of stderr, kept for the dpkg-source failure + // classification. + let stderr_capture = Arc::new(std::sync::Mutex::new(String::new())); + let status = match sink { - None => cmd - .status() - .map_err(|e| format!("failed to run '{}': {}", program, e))?, + None => cmd.status().map_err(|e| CommandFailure { + error: format!("failed to run '{}': {}", program, e).into(), + stderr: String::new(), + })?, Some(sink) => { cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - let mut child = cmd - .spawn() - .map_err(|e| format!("failed to run '{}': {}", program, e))?; + let mut child = cmd.spawn().map_err(|e| CommandFailure { + error: format!("failed to run '{}': {}", program, e).into(), + stderr: String::new(), + })?; let stdout = child.stdout.take(); let stderr = child.stderr.take(); @@ -642,37 +838,89 @@ fn run_command( // approximate (channel arrival order), acceptable for display. let out_sink = sink.clone(); let err_sink = sink.clone(); + let err_capture = Arc::clone(&stderr_capture); let out_thread = std::thread::spawn(move || { if let Some(out) = stdout { pump(out, Stream::Stdout, &*out_sink); } }); let err_thread = std::thread::spawn(move || { - if let Some(err) = stderr { - pump(err, Stream::Stderr, &*err_sink); + if let Some(stderr) = stderr { + pump( + stderr, + Stream::Stderr, + &CapturingSink { + inner: err_sink, + capture: err_capture, + }, + ); } }); let _ = out_thread.join(); let _ = err_thread.join(); - child - .wait() - .map_err(|e| format!("failed to wait for '{}': {}", program, e))? + child.wait().map_err(|e| CommandFailure { + error: format!("failed to wait for '{}': {}", program, e).into(), + stderr: String::new(), + })? } }; if !status.success() { - return Err(format!( - "'{} {}' failed with status: {}", - program, - args.join(" "), - status - ) - .into()); + return Err(CommandFailure { + error: format!( + "'{} {}' failed with status: {}", + program, + args.join(" "), + status + ) + .into(), + stderr: std::mem::take(&mut stderr_capture.lock().unwrap_or_else(|e| e.into_inner())), + }); } Ok(()) } +/// Run a build command, discarding the captured stderr. +fn run_command( + cwd: &Path, + program: &str, + args: &[&str], + env: &BTreeMap, + sink: Option<&Arc>, +) -> Result<(), Box> { + run_command_capturing(cwd, program, args, env, sink).map_err(|failure| failure.error) +} + +/// A [`LineSink`] forwarding every line to the live view while appending +/// stderr lines to the classification buffer. +struct CapturingSink { + inner: Arc, + capture: Arc>, +} + +const STDERR_CAPTURE_LIMIT: usize = 64 * 1024; + +impl LineSink for CapturingSink { + fn line(&self, stream: Stream, line: &str) { + if stream == Stream::Stderr { + let mut buffer = self.capture.lock().unwrap_or_else(|e| e.into_inner()); + buffer.push_str(line); + buffer.push('\n'); + // Keep the tail: dpkg-source reports the offending path last. + if buffer.len() > STDERR_CAPTURE_LIMIT { + let cut = buffer.len() - STDERR_CAPTURE_LIMIT; + let start = buffer[cut..] + .find('\n') + .map(|index| cut + index + 1) + .unwrap_or(buffer.len()); + buffer.drain(..start); + } + } + self.inner.line(stream, line); + } +} + #[cfg(test)] mod tests { use super::*; @@ -757,6 +1005,118 @@ mod tests { assert_eq!(entries.len(), 1); assert_eq!(entries[0].name, "hello.tar.xz"); } + + /// Real-shape `dpkg-source -b` stderr excerpts and their class. + #[test] + fn dpkg_source_failure_classification() { + // Vendored dependencies diverged from the orig-vendor component + // (dpkg names the offending paths before the summary line). + let vendor = concat!( + "dpkg-source: info: building mytool using existing \ + ./mytool_0.1.0.orig.tar.xz\n", + "dpkg-source: info: using source format version 3.0 (quilt)\n", + "dpkg-source: error: cannot represent change to \ + vendor/libc/src/unix/linux_like/mod.rs: binary file contents \ + changed\n", + "dpkg-source: error: unrepresentable changes to source\n", + ); + assert_eq!( + classify_dpkg_source_failure(vendor), + DpkgSourceFailure::VendorDrift + ); + + // A missing orig-vendor component makes the vendored files look + // "new"; still vendor drift. + let missing_component = concat!( + "dpkg-source: error: cannot represent change to \ + vendor/libc/Cargo.toml: new file is binary\n", + "dpkg-source: error: unrepresentable changes to source\n", + ); + assert_eq!( + classify_dpkg_source_failure(missing_component), + DpkgSourceFailure::VendorDrift + ); + + // The component named in the error is vendor drift too. + assert_eq!( + classify_dpkg_source_failure( + "dpkg-source: error: orig-vendor component tarball \ + checksum mismatch\n", + ), + DpkgSourceFailure::VendorDrift + ); + + // Upstream tree drift outside vendor/. + let upstream = concat!( + "dpkg-source: error: cannot represent change to \ + assets/logo.png: binary file contents changed\n", + "dpkg-source: error: unrepresentable changes to source\n", + ); + assert_eq!( + classify_dpkg_source_failure(upstream), + DpkgSourceFailure::UpstreamDrift + ); + assert_eq!( + classify_dpkg_source_failure("dpkg-source: error: unrepresentable changes to source\n"), + DpkgSourceFailure::UpstreamDrift + ); + + // Anything else. + assert_eq!( + classify_dpkg_source_failure( + "dpkg-source: error: syntax error in debian/control at line 3\n" + ), + DpkgSourceFailure::Other + ); + assert_eq!(classify_dpkg_source_failure(""), DpkgSourceFailure::Other); + } + + /// The user-facing wording of the classified failures. + #[test] + fn dpkg_source_failure_messages() { + let vendor = "dpkg-source: error: cannot represent change to \ + vendor/serde/src/de/mod.rs: binary file contents changed\n\ + dpkg-source: error: unrepresentable changes to source\n"; + let error = dpkg_source_failure_error( + DpkgSourceFailure::VendorDrift, + vendor, + "'dpkg-source -I -i -b .' failed with status: exit status: 2".into(), + ); + let message = error.to_string(); + assert!( + message.contains("vendored dependencies changed"), + "{message}" + ); + assert!(message.contains("orig-vendor"), "{message}"); + // The offending dpkg line travels along as the detail. + assert!(message.contains("vendor/serde/src/de/mod.rs"), "{message}"); + // Regression: the typed error must SURVIVE the message building, or + // the re-vendor retry hook never fires. + assert!(error.downcast_ref::().is_some()); + + let upstream_error = dpkg_source_failure_error( + DpkgSourceFailure::UpstreamDrift, + "dpkg-source: error: unrepresentable changes to source\n", + "'dpkg-source -I -i -b .' failed with status: exit status: 2".into(), + ); + let message = upstream_error.to_string(); + assert!( + message.contains("drifted beyond the orig snapshot"), + "{message}" + ); + assert!(message.contains("pkh chlog"), "{message}"); + + // Other: the raw command error passes through untouched. + let other = dpkg_source_failure_error(DpkgSourceFailure::Other, "", "boom".into()); + assert_eq!(other.to_string(), "boom"); + + // A vendor-drift error (with or without detail) is recognized by + // the retry hook. + for detail in [None, Some("dpkg-source: error: nope".to_string())] { + let boxed: Box = Box::new(VendorDriftError { detail }); + assert!(boxed.downcast_ref::().is_some()); + } + } } /// Differential tests: build synthetic (or real archive) source packages diff --git a/src/main.rs b/src/main.rs index cdde41a..e42d2ff 100644 --- a/src/main.rs +++ b/src/main.rs @@ -119,7 +119,29 @@ fn main() { .help("Target series (default: the development series of --dist)"), ) .arg(arg!(--release "Write the --series into debian/changelog instead of UNRELEASED").required(false)) - .arg(arg!(--native "Use the 3.0 (native) source format (no orig tarball)").required(false)) + .arg(arg!(--native "Use the 3.0 (native) source format (default for a new project skeleton: no orig tarball)").required(false)) + .arg( + clap::Arg::new("quilt") + .long("quilt") + .action(clap::ArgAction::SetTrue) + .conflicts_with("native") + .help("Use the 3.0 (quilt) source format with an orig tarball (default when packaging an existing project)"), + ) + .arg( + clap::Arg::new("orig_from") + .long("orig-from") + .value_name("MODE") + .value_parser(["release", "git", "path", "snapshot"]) + .conflicts_with("native") + .help("How to produce the orig tarball (quilt only): release (download the forge tarball of the tag; network), git (git archive of the tag), path (repack --orig-path), snapshot (tar the working tree). Default: git on a tag, snapshot otherwise"), + ) + .arg( + clap::Arg::new("orig_path") + .long("orig-path") + .value_name("FILE|URL") + .conflicts_with("native") + .help("Tarball used by --orig-from path: a local file or an http(s) URL (.tar, .tar.gz, .tgz, .tar.bz2, .tbz2, .tar.xz), repacked to the orig"), + ) .arg( clap::Arg::new("no_git") .long("no-git") @@ -273,6 +295,12 @@ fn main() { .get_one::("native") .copied() .unwrap_or(false), + quilt: sub_matches + .get_one::("quilt") + .copied() + .unwrap_or(false), + orig_from: sub_matches.get_one::("orig_from").cloned(), + orig_path: sub_matches.get_one::("orig_path").cloned(), git: !sub_matches .get_one::("no_git") .copied() diff --git a/src/new/debian.rs b/src/new/debian.rs index 212793c..e77948e 100644 --- a/src/new/debian.rs +++ b/src/new/debian.rs @@ -12,12 +12,13 @@ use chrono::Datelike; use tar::Builder; use xz2::write::XzEncoder; -use super::options::NewOptions; +use super::options::{NewOptions, SourceFormat}; use super::templates::{OutputFile, Template}; -/// `3.0 (quilt)` source format, the pkh new default. +/// `3.0 (quilt)` source format, the default when packaging an existing +/// project. pub const SOURCE_FORMAT_QUILT: &str = "3.0 (quilt)"; -/// `3.0 (native)` source format, selected by `--native`. +/// `3.0 (native)` source format, the default for a fresh skeleton. pub const SOURCE_FORMAT_NATIVE: &str = "3.0 (native)"; /// The three source formats pkh knows how to build. @@ -59,7 +60,7 @@ pub fn files(opts: &NewOptions, template: &dyn Template) -> Vec { copyright(opts), debian_gitignore(opts), ]; - if !opts.native { + if opts.source_format == SourceFormat::Quilt { files.push(local_options()); } if opts.autopkgtest { @@ -94,19 +95,13 @@ fn autopkgtest_smoke(opts: &NewOptions) -> OutputFile { ) } -/// `debian/source/format`: `3.0 (quilt)` by default, `3.0 (native)` with -/// `--native`. +/// `debian/source/format`: `3.0 (native)` for a skeleton by default, +/// `3.0 (quilt)` for an existing project; either can be forced with +/// `--native` / `--quilt`. fn source_format(opts: &NewOptions) -> OutputFile { OutputFile::new( "debian/source/format", - format!( - "{}\n", - if opts.native { - SOURCE_FORMAT_NATIVE - } else { - SOURCE_FORMAT_QUILT - } - ), + format!("{}\n", opts.source_format.deb_string()), ) } @@ -369,6 +364,20 @@ pub fn create_orig_tarball( tree: &Path, name: &str, upstream_version: &str, +) -> Result> { + create_orig_tarball_excluding(tree, name, upstream_version, false) +} + +/// [`create_orig_tarball`] with the generated `vendor/` directory of a +/// vendored rust package excluded from the snapshot: its contents travel in +/// the separate `_.orig-vendor.tar.xz` component instead (see +/// [`super::orig`]), so they can be regenerated independently of the +/// upstream sources. +pub fn create_orig_tarball_excluding( + tree: &Path, + name: &str, + upstream_version: &str, + exclude_vendor: bool, ) -> Result> { let tarball_path = orig_tarball_path(tree, name, upstream_version).ok_or_else(|| { format!( @@ -393,7 +402,8 @@ pub fn create_orig_tarball( let prefix = format!("{name}-{upstream_version}"); // The single top-level directory dpkg-source expects. builder.append_dir(&prefix, tree)?; - append_tree(&mut builder, tree, &prefix, 0)?; + let top_excludes: &[&str] = if exclude_vendor { &["vendor"] } else { &[] }; + append_tree(&mut builder, tree, &prefix, 0, ORIG_EXCLUDE, top_excludes)?; builder .finish() @@ -407,12 +417,15 @@ pub fn create_orig_tarball( } /// Recursively append `dir` to the archive under `archive_path`, skipping -/// the [`ORIG_EXCLUDE`] names and non-regular files. -fn append_tree( +/// non-regular files, the names of `excludes` at any depth, the `debian/` +/// directory and the names of `top_excludes` at the top level (depth 0). +pub(crate) fn append_tree( builder: &mut Builder>, dir: &Path, archive_path: &str, depth: usize, + excludes: &[&str], + top_excludes: &[&str], ) -> Result<(), Box> { let mut entries: Vec = std::fs::read_dir(dir)?.collect::>()?; entries.sort_by_key(|entry| entry.file_name()); @@ -422,10 +435,10 @@ fn append_tree( let file_name = entry.file_name(); let name = file_name.to_string_lossy().into_owned(); - if depth == 0 && name == "debian" { + if depth == 0 && (name == "debian" || top_excludes.contains(&name.as_str())) { continue; } - if ORIG_EXCLUDE.contains(&name.as_str()) { + if excludes.contains(&name.as_str()) { continue; } @@ -434,7 +447,14 @@ fn append_tree( .map_err(|e| format!("cannot stat '{}': {}", path.display(), e))?; if metadata.is_dir() { builder.append_dir(&entry_archive_path, &path)?; - append_tree(builder, &path, &entry_archive_path, depth + 1)?; + append_tree( + builder, + &path, + &entry_archive_path, + depth + 1, + excludes, + top_excludes, + )?; } else if metadata.is_file() { // The mode (including the exec bit) travels through the header. let mut header = tar::Header::new_gnu(); @@ -502,7 +522,8 @@ mod tests { series: "resolute".into(), release: false, depends: Vec::new(), - native: false, + source_format: SourceFormat::Quilt, + orig: None, git: true, autopkgtest: false, pkg_config: false, @@ -528,7 +549,7 @@ mod tests { ); let native = NewOptions { - native: true, + source_format: SourceFormat::Native, ..opts() }; let files = super::files( @@ -799,6 +820,44 @@ mod tests { assert!(err.to_string().contains("already exists")); } + /// The vendored-rust variant excludes the top-level `vendor/` (it + /// travels in the orig-vendor component) but keeps unrelated trees. + #[test] + fn orig_tarball_vendor_exclusion() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(tree.join("vendor/serde/src")).unwrap(); + std::fs::create_dir_all(tree.join("src/vendor")).unwrap(); + std::fs::write(tree.join("vendor/serde/src/lib.rs"), "code").unwrap(); + std::fs::write(tree.join("src/vendor/mod.rs"), "code").unwrap(); + std::fs::write(tree.join("Cargo.toml"), "[package]").unwrap(); + + let tarball = create_orig_tarball_excluding(&tree, "mytool", "0.1.0", true).unwrap(); + let mut archive = tar::Archive::new(xz2::read::XzDecoder::new( + std::fs::File::open(&tarball).unwrap(), + )); + let names: Vec = archive + .entries() + .unwrap() + .map(|e| e.unwrap().path().unwrap().to_string_lossy().into_owned()) + .collect(); + // The generated vendored tree is out... + assert!( + !names.iter().any(|n| n.starts_with("mytool-0.1.0/vendor")), + "{names:?}" + ); + // ...an unrelated nested vendor/ stays in... + assert!( + names.iter().any(|n| n == "mytool-0.1.0/src/vendor/mod.rs"), + "{names:?}" + ); + // ...and normal files are unaffected. + assert!( + names.iter().any(|n| n == "mytool-0.1.0/Cargo.toml"), + "{names:?}" + ); + } + #[test] fn write_files_sets_exec_bit_and_parents() { let dir = tempfile::tempdir().unwrap(); diff --git a/src/new/mod.rs b/src/new/mod.rs index 66583a0..383b152 100644 --- a/src/new/mod.rs +++ b/src/new/mod.rs @@ -3,8 +3,9 @@ //! //! This module orchestrates a scaffold run: target directory checks, project //! detection, in-memory rendering of every file (all-or-nothing write), the -//! template post-write hook (e.g. `cargo vendor`), orig tarball creation, -//! git initialization, structural verification and the next-steps message. +//! template post-write hook (e.g. `cargo vendor`), orig tarball creation +//! (from the origin the run decided on — see [`origin`] and [`orig`]), git +//! initialization, structural verification and the next-steps message. //! The interactive wizard ([`questions`]) fills a [`options::NewCli`] from //! its answers on a TTY and reuses [`options::resolve`] as the single source //! of truth for defaults and validation; without a TTY the same resolution @@ -14,6 +15,8 @@ pub mod debian; pub mod detect; pub mod git; pub mod options; +pub mod orig; +pub mod origin; pub mod questions; pub mod templates; pub mod verify; @@ -23,7 +26,7 @@ use std::time::Duration; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; -use options::NewOptions; +use options::{NewOptions, SourceFormat}; use templates::{OutputFile, ScaffoldOutcome}; /// Scaffold a full Debian source tree from `opts`. @@ -64,8 +67,8 @@ pub fn scaffold( pb.finish_and_clear(); multi.remove(&pb); - if result.is_ok() { - print_success(&opts); + if let Ok(outcome) = &result { + print_success(&opts, outcome); } result } @@ -127,7 +130,7 @@ fn scaffold_steps(opts: &NewOptions, pb: &ProgressBar) -> Result Result Result NewOptions { + // Mirror the resolve() derivation: skeletons are native by default, + // existing projects quilt with a working-tree snapshot orig. + let (source_format, orig) = match source_dir { + SourceDir::Skeleton => (SourceFormat::Native, None), + SourceDir::Here | SourceDir::Path(_) => { + (SourceFormat::Quilt, Some(OrigOrigin::Snapshot)) + } + }; NewOptions { name: name.to_string(), template, @@ -265,7 +298,8 @@ mod tests { series: "resolute".into(), release: false, depends: Vec::new(), - native: false, + source_format, + orig, git: false, autopkgtest: false, pkg_config: false, @@ -305,7 +339,6 @@ mod tests { "debian/rules", "debian/copyright", "debian/source/format", - "debian/source/local-options", "debian/.gitignore", "debian/install", "mytool.sh", @@ -340,15 +373,14 @@ mod tests { assert_eq!(version, "0.1.0-1"); assert_eq!(distribution, "UNRELEASED"); - // source/format + local-options. + // A fresh skeleton is 3.0 (native) by default: no local-options and + // no orig tarball anywhere. assert_eq!( std::fs::read_to_string(tree.join("debian/source/format")).unwrap(), - "3.0 (quilt)\n" - ); - assert_eq!( - std::fs::read_to_string(tree.join("debian/source/local-options")).unwrap(), - "single-debian-patch\n" + "3.0 (native)\n" ); + assert!(!tree.join("debian/source/local-options").exists()); + assert!(!dir.path().join("mytool_0.1.0.orig.tar.xz").exists()); // install mapping. assert_eq!( @@ -360,6 +392,28 @@ mod tests { let gitignore = std::fs::read_to_string(tree.join(".gitignore")).unwrap(); assert!(gitignore.contains("*.deb")); assert!(gitignore.contains("target/")); + } + + /// A skeleton forced to quilt keeps the snapshot behavior: orig tarball + /// with the skeleton files, `debian/` excluded, local-options present. + #[test] + #[serial] + fn scaffold_skeleton_forced_quilt_snapshots_the_tree() { + let dir = tempdir().unwrap(); + let mut o = opts(TemplateId::Shell, "mytool", SourceDir::Skeleton); + o.source_format = SourceFormat::Quilt; + o.orig = Some(OrigOrigin::Snapshot); + scaffold_in(dir.path(), o).unwrap(); + + let tree = dir.path().join("mytool"); + assert_eq!( + std::fs::read_to_string(tree.join("debian/source/format")).unwrap(), + "3.0 (quilt)\n" + ); + assert_eq!( + std::fs::read_to_string(tree.join("debian/source/local-options")).unwrap(), + "single-debian-patch\n" + ); // Orig tarball: contains the skeleton file, excludes debian/. let tarball = dir.path().join("mytool_0.1.0.orig.tar.xz"); @@ -403,29 +457,11 @@ mod tests { ); assert_eq!(control.binaries[0].get("Architecture"), Some("all")); // No install file, no build-system skeleton: the README stub only. + // A native skeleton carries no orig tarball: the README simply + // lives in the tree. assert!(!tree.join("debian/install").exists()); assert!(tree.join("README").exists()); - // The tarball excludes debian/ but carries the README. - let tarball = dir.path().join("metapkg_0.1.0.orig.tar.xz"); - let mut archive = tar::Archive::new(xz2::read::XzDecoder::new( - std::fs::File::open(&tarball).unwrap(), - )); - let names: Vec = archive - .entries() - .unwrap() - .map(|entry| { - entry - .unwrap() - .path() - .unwrap() - .to_string_lossy() - .into_owned() - }) - .collect(); - assert!( - names.iter().any(|n| n == "metapkg-0.1.0/README"), - "{names:?}" - ); + assert!(!dir.path().join("metapkg_0.1.0.orig.tar.xz").exists()); // Empty base flavor: no depends, no Depends field. let dir = tempdir().unwrap(); @@ -527,14 +563,13 @@ mod tests { .unwrap_err(); assert!(err.to_string().contains("not empty"), "{err}"); - // Existing orig tarball: nothing gets written. + // Existing orig tarball (quilt only): nothing gets written. let dir = tempdir().unwrap(); std::fs::write(dir.path().join("mytool_0.1.0.orig.tar.xz"), b"old").unwrap(); - let err = scaffold_in( - dir.path(), - opts(TemplateId::Shell, "mytool", SourceDir::Skeleton), - ) - .unwrap_err(); + let mut o = opts(TemplateId::Shell, "mytool", SourceDir::Skeleton); + o.source_format = SourceFormat::Quilt; + o.orig = Some(OrigOrigin::Snapshot); + let err = scaffold_in(dir.path(), o).unwrap_err(); assert!(err.to_string().contains("already exists"), "{err}"); assert!(!dir.path().join("mytool/debian/control").exists()); @@ -552,13 +587,17 @@ mod tests { assert!(err.to_string().contains("does not exist"), "{err}"); } + /// The skeleton default is `3.0 (native)`: no orig tarball, no + /// `debian/source/local-options`. #[test] #[serial] - fn scaffold_native_skips_tarball_and_local_options() { + fn scaffold_skeleton_defaults_to_native() { let dir = tempdir().unwrap(); - let mut o = opts(TemplateId::Shell, "nativepkg", SourceDir::Skeleton); - o.native = true; - scaffold_in(dir.path(), o).unwrap(); + scaffold_in( + dir.path(), + opts(TemplateId::Shell, "nativepkg", SourceDir::Skeleton), + ) + .unwrap(); let tree = dir.path().join("nativepkg"); assert!(!dir.path().join("nativepkg_0.1.0.orig.tar.xz").exists()); @@ -569,17 +608,17 @@ mod tests { ); } - /// End-to-end rust skeleton: the vendoring hook runs before the orig - /// tarball is created, so `.cargo/` (and `vendor/` when dependencies - /// exist) travel inside it. The vendoring step needs host cargo; on a - /// cargo-less host the scaffold still succeeds with a warning and a - /// `vendoring_failed` outcome. Keyed against the `RUSTUP_TOOLCHAIN` - /// tests of the rust template: they mutate the process-global - /// environment the cargo shim would pick up mid-vendoring. + /// End-to-end rust skeleton: the vendoring hook runs over the tree + /// (native format: everything simply lives in the tree, no orig + /// tarball). The vendoring step needs host cargo; on a cargo-less host + /// the scaffold still succeeds with a warning and a `vendoring_failed` + /// outcome. Keyed against the `RUSTUP_TOOLCHAIN` tests of the rust + /// template: they mutate the process-global environment the cargo shim + /// would pick up mid-vendoring. #[test] #[serial] #[serial(RUSTUP_TOOLCHAIN)] - fn scaffold_rust_skeleton_vendors_before_tarball() { + fn scaffold_rust_skeleton_vendors_into_the_tree() { let dir = tempdir().unwrap(); let outcome = scaffold_in( dir.path(), @@ -589,6 +628,9 @@ mod tests { let has_cargo = crate::new::templates::find_on_path("cargo").is_some(); assert_eq!(outcome.vendoring_failed, !has_cargo); + // Native skeleton: no orig tarball at all. + assert_eq!(outcome.orig_origin, None); + assert!(!dir.path().join("mytool_0.1.0.orig.tar.xz").exists()); let tree = dir.path().join("mytool"); assert!(tree.join("Cargo.toml").exists()); @@ -613,14 +655,60 @@ mod tests { Some("debhelper-compat (= 13),\ncargo:native,\nrustc:native") ); - // The offline config exists when host cargo vendored the skeleton, - // and both it and the skeleton land inside the orig tarball. + // The offline config exists in the tree when host cargo vendored + // the skeleton. if has_cargo { let config = std::fs::read_to_string(tree.join(".cargo/config.toml")).unwrap(); assert!(config.contains("[source.crates-io]"), "{config}"); assert!(config.contains("[net]\noffline = true"), "{config}"); } + } + /// End-to-end vendored rust quilt package: the vendoring hook creates + /// `vendor/`, the main orig excludes it, the dpkg upstream component + /// `orig-vendor` carries it, and the real source build + /// (`run_source_build`) lists BOTH tarballs in the `.dsc` and succeeds. + /// Needs host cargo with a working crates.io sync (skipped gracefully + /// when either is unavailable) and the local dpkg tools. + #[test] + #[serial] + #[serial(RUSTUP_TOOLCHAIN)] + fn scaffold_rust_quilt_with_deps_vendors_into_a_component() { + let dir = tempdir().unwrap(); + let source = dir.path().join("mytool"); + std::fs::create_dir_all(source.join("src")).unwrap(); + // `libc` resolves from the host's cargo registry cache; vendoring + // it needs one crates.io index sync. + std::fs::write( + source.join("Cargo.toml"), + "[package]\n\ + name = \"mytool\"\n\ + version = \"0.1.0\"\n\ + edition = \"2021\"\n\ + \n\ + [dependencies]\n\ + libc = \"0.2\"\n", + ) + .unwrap(); + std::fs::write(source.join("src/main.rs"), "fn main() {}\n").unwrap(); + + let mut o = opts(TemplateId::Rust, "mytool", SourceDir::Path(source.clone())); + o.source_format = SourceFormat::Quilt; + o.orig = Some(OrigOrigin::Snapshot); + let outcome = scaffold_in(dir.path(), o).unwrap(); + + if crate::new::templates::find_on_path("cargo").is_none() || outcome.vendoring_failed { + // No cargo on this host or the crates.io sync failed: the + // vendoring guarantees of this test cannot hold. + log::warn!("cargo/crates.io unavailable; skipping the vendored component checks"); + return; + } + assert_eq!( + outcome.orig_origin, + Some("working tree snapshot".to_string()) + ); + + // The main orig excludes vendor/ but carries the upstream files. let tarball = dir.path().join("mytool_0.1.0.orig.tar.xz"); let mut archive = tar::Archive::new(xz2::read::XzDecoder::new( std::fs::File::open(&tarball).unwrap(), @@ -641,21 +729,58 @@ mod tests { names.iter().any(|n| n == "mytool-0.1.0/Cargo.toml"), "{names:?}" ); + assert!(!names.iter().any(|n| n.contains("vendor")), "{names:?}"); + + // The component carries vendor/ under a top-level vendor/ dir. + let component = dir.path().join("mytool_0.1.0.orig-vendor.tar.xz"); + assert!(component.exists()); + let mut archive = tar::Archive::new(xz2::read::XzDecoder::new( + std::fs::File::open(&component).unwrap(), + )); + let names: Vec = archive + .entries() + .unwrap() + .map(|entry| { + entry + .unwrap() + .path() + .unwrap() + .to_string_lossy() + .into_owned() + }) + .collect(); assert!( - names.iter().any(|n| n == "mytool-0.1.0/src/main.rs"), + names.iter().any(|n| n.starts_with("vendor/libc/")), "{names:?}" ); - if has_cargo { - assert!( - names.iter().any(|n| n == "mytool-0.1.0/.cargo/config.toml"), - "{names:?}" - ); - } - assert!(!names.iter().any(|n| n.contains("debian")), "{names:?}"); + + // The real source build: the .dsc references both tarballs. + let output = crate::build::run_source_build( + &source, + &crate::build::SourceBuildOptions::default(), + None, + ) + .unwrap(); + let dsc = std::fs::read_to_string(&output.dsc).unwrap(); + assert!(dsc.contains("mytool_0.1.0.orig.tar.xz"), "{dsc}"); + assert!(dsc.contains("mytool_0.1.0.orig-vendor.tar.xz"), "{dsc}"); + assert!( + output + .tarballs + .iter() + .any(|t| t.ends_with("mytool_0.1.0.orig.tar.xz")) + ); + assert!( + output + .tarballs + .iter() + .any(|t| t.ends_with("mytool_0.1.0.orig-vendor.tar.xz")) + ); + assert!(!output.signed); } - /// End-to-end python skeleton: pyproject-based Build-Depends and the - /// module skeleton inside the orig tarball. + /// End-to-end python skeleton: pyproject-based Build-Depends (native + /// skeleton: the upstream files live in the tree, no orig tarball). #[test] #[serial] fn scaffold_python_skeleton_tree() { @@ -679,44 +804,22 @@ mod tests { let rules = std::fs::read_to_string(tree.join("debian/rules")).unwrap(); assert!(rules.contains("%:\n\tdh $@ --with python3 --buildsystem=pybuild\n")); - let tarball = dir.path().join("mytool_0.1.0.orig.tar.xz"); - let mut archive = tar::Archive::new(xz2::read::XzDecoder::new( - std::fs::File::open(&tarball).unwrap(), - )); - let names: Vec = archive - .entries() - .unwrap() - .map(|entry| { - entry - .unwrap() - .path() - .unwrap() - .to_string_lossy() - .into_owned() - }) - .collect(); - assert!( - names.iter().any(|n| n == "mytool-0.1.0/pyproject.toml"), - "{names:?}" - ); - assert!( - names.iter().any(|n| n == "mytool-0.1.0/mytool/__init__.py"), - "{names:?}" - ); + assert!(tree.join("pyproject.toml").exists()); + assert!(tree.join("mytool/__init__.py").exists()); + assert!(!dir.path().join("mytool_0.1.0.orig.tar.xz").exists()); } - /// End-to-end: the scaffolded shell tree passes the real source build - /// (`dpkg-source` and friends, same prerequisites as the differential - /// tests). + /// End-to-end: a quilt tree (Here mode over an existing source) passes + /// the real source build (`dpkg-source` and friends, same prerequisites + /// as the differential tests). #[test] #[serial] fn scaffold_then_source_build_produces_artifacts() { let dir = tempdir().unwrap(); - scaffold_in( - dir.path(), - opts(TemplateId::Shell, "mytool", SourceDir::Skeleton), - ) - .unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(&tree).unwrap(); + std::fs::write(tree.join("run.sh"), "#!/bin/sh\necho hi\n").unwrap(); + scaffold_in(&tree, opts(TemplateId::Shell, "mytool", SourceDir::Here)).unwrap(); let output = crate::build::run_source_build( &dir.path().join("mytool"), @@ -736,4 +839,36 @@ mod tests { // UNRELEASED: nothing is signed. assert!(!output.signed); } + + /// End-to-end native: a self-authored skeleton (3.0 (native), no orig + /// tarball) builds into a .dsc without any tarball at all. + #[test] + #[serial] + fn scaffold_native_then_source_build_needs_no_tarball() { + let dir = tempdir().unwrap(); + scaffold_in( + dir.path(), + opts(TemplateId::Shell, "mytool", SourceDir::Skeleton), + ) + .unwrap(); + + let output = crate::build::run_source_build( + &dir.path().join("mytool"), + &crate::build::SourceBuildOptions::default(), + None, + ) + .unwrap(); + assert!(output.dsc.exists(), "{:?} missing", output.dsc); + assert!(output.changes.exists(), "{:?} missing", output.changes); + // 3.0 (native): one self-contained source tarball (debian/ inside), + // but no ORIG tarball. + assert_eq!(output.tarballs.len(), 1, "{:?}", output.tarballs); + assert!( + !output.tarballs[0] + .file_name() + .is_some_and(|name| name.to_string_lossy().contains("orig")), + "{:?}", + output.tarballs + ); + } } diff --git a/src/new/options.rs b/src/new/options.rs index f1cf27c..ee418f5 100644 --- a/src/new/options.rs +++ b/src/new/options.rs @@ -15,6 +15,8 @@ use crate::debian::DebianVersion; use crate::debian::deps::{Deps, ParseOpts}; use crate::distro_info; use crate::new::detect::{self, Detection}; +use crate::new::origin::{Forge, GitOrigin}; +use crate::new::templates; /// Build systems / project kinds `pkh new` knows about. /// @@ -133,6 +135,82 @@ pub enum SourceDir { Path(PathBuf), } +impl SourceDir { + /// The directory being packaged, when it already exists on disk (`None` + /// for the skeleton mode, whose sources are rendered in memory). + pub fn existing_dir(&self, cwd: &Path) -> Option { + match self { + SourceDir::Skeleton => None, + SourceDir::Here => Some(cwd.to_path_buf()), + SourceDir::Path(path) => Some(path.clone()), + } + } +} + +/// Debian source format of the scaffolded package: an explicit, derived +/// choice rather than a boolean afterthought. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceFormat { + /// `3.0 (native)`: no orig tarball at all. The default for a fresh, + /// self-authored skeleton. + Native, + /// `3.0 (quilt)`: upstream sources in the orig tarball, packaging in + /// `debian/`. The default when packaging an existing project. + Quilt, +} + +impl SourceFormat { + /// The value written to `debian/source/format`. + pub fn deb_string(&self) -> &'static str { + match self { + SourceFormat::Native => super::debian::SOURCE_FORMAT_NATIVE, + SourceFormat::Quilt => super::debian::SOURCE_FORMAT_QUILT, + } + } +} + +/// How the orig tarball of a quilt package is produced. The right answer +/// depends on where the upstream sources come from, so the wizard offers +/// them pre-ordered by what git origin detection found (release download +/// and `git archive` only exist when HEAD sits exactly on a tag). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum OrigOrigin { + /// Download the upstream release tarball of `tag` from the forge of the + /// `origin` remote and repack it (explicit network consent). + Release { + /// Release tag the tarball is downloaded for. + tag: String, + /// Forge the tarball is downloaded from. + forge: Forge, + }, + /// `git archive` the tag HEAD sits on: offline and deterministic. + GitArchive { + /// Release tag archived into the tarball. + tag: String, + }, + /// Repack a tarball the user provides (a local path or an http(s) URL). + Provided { + /// Path or URL of the user-provided tarball. + source: String, + }, + /// Snapshot the working tree into the orig tarball (the only option for + /// a skeleton, and the fallback when the upstream history is unknown). + Snapshot, +} + +impl OrigOrigin { + /// Human-readable description used in the wizard summary and the + /// post-scaffold report. + pub fn label(&self) -> String { + match self { + OrigOrigin::Release { tag, .. } => format!("release download ({tag})"), + OrigOrigin::GitArchive { tag } => format!("git archive ({tag})"), + OrigOrigin::Provided { source } => format!("user tarball ({source})"), + OrigOrigin::Snapshot => "working tree snapshot".to_string(), + } + } +} + /// Upstream license of the package: a curated SPDX list plus a free-text /// fallback for anything else (including "unknown" until the user picks one). #[derive(Debug, Clone, PartialEq, Eq)] @@ -242,8 +320,13 @@ pub struct NewOptions { /// Runtime Depends clauses of the metapackage flavor (canonically /// rendered; empty for every other flavor). pub depends: Vec, - /// Use the `3.0 (native)` source format (no orig tarball). - pub native: bool, + /// Debian source format: `3.0 (native)` for a fresh skeleton by + /// default, `3.0 (quilt)` when packaging an existing project; either + /// can be forced with `--native` / `--quilt`. + pub source_format: SourceFormat, + /// How the orig tarball is produced (quilt only; `None` with the native + /// format, which has no orig tarball). + pub orig: Option, /// Initialize a git repository (gitignores are written regardless). pub git: bool, /// Write the autopkgtest smoke test (`debian/tests/control` + @@ -320,6 +403,12 @@ pub struct NewCli { pub release: bool, /// `--native`. pub native: bool, + /// `--quilt` (mutually exclusive with `--native`). + pub quilt: bool, + /// `--orig-from `. + pub orig_from: Option, + /// `--orig-path ` (required by `--orig-from path`). + pub orig_path: Option, /// True unless `--no-git`. pub git: bool, /// `--defaults`. @@ -604,11 +693,65 @@ pub async fn resolve(cli: NewCli) -> Result { // Everything below has a default and is validated as it is resolved. validate_source_name(&name)?; - let upstream_version = cli.upstream_version.unwrap_or_else(|| "0.1.0".to_string()); + // Source format: explicit flags win (--native / --quilt, mutually + // exclusive), otherwise the mode decides — a fresh, self-authored + // skeleton is native by default, an existing project is quilt. + let source_format = match (cli.native, cli.quilt) { + (true, true) => { + return Err("--native and --quilt are mutually exclusive".to_string()); + } + (true, false) => SourceFormat::Native, + (false, true) => SourceFormat::Quilt, + (false, false) => match source_dir { + SourceDir::Skeleton => SourceFormat::Native, + SourceDir::Here | SourceDir::Path(_) => SourceFormat::Quilt, + }, + }; + + // Git origin state of the packaged directory: needed for the version + // default (tag / +git scheme) and for the orig-tarball plan. Probing is + // purely local (git config and refs, no network). + let existing_dir = source_dir.existing_dir(&std::env::current_dir().unwrap_or_default()); + let needs_origin = cli.upstream_version.is_none() + || (source_format == SourceFormat::Quilt + && (cli.orig_from.is_some() || existing_dir.is_some())); + let origin = if needs_origin { + existing_dir.as_deref().and_then(GitOrigin::detect) + } else { + None + }; + + // Upstream version default: probed project version, else the tag HEAD + // sits on, else `+git.`, else 0.1.0. + let upstream_version = match cli.upstream_version { + Some(ref version) => version.clone(), + None => { + let probed = template + .and_then(templates::get) + .and_then(|t| t.probe(existing_dir.as_deref()?)) + .and_then(|probe| probe.version) + .filter(|version| { + validate_upstream_version(version, cli.revision.unwrap_or(1)).is_ok() + }); + let derived = probed + .or_else(|| origin.as_ref().and_then(GitOrigin::head_tag_version)) + .or_else(|| origin.as_ref().and_then(GitOrigin::git_version)) + .unwrap_or_else(|| "0.1.0".to_string()); + if derived != "0.1.0" { + log::info!("No --upstream-version given, defaulting to '{derived}'"); + } + derived + } + }; let revision = cli.revision.unwrap_or(1); validate_upstream_version(&upstream_version, revision) .map_err(|e| format!("Invalid upstream version: {e}"))?; + // Orig-tarball plan (quilt only). Without an explicit --orig-from the + // non-interactive default never touches the network: `git archive` of + // the tag when HEAD sits on one, a working-tree snapshot otherwise. + let orig = resolve_orig_origin(&cli, &source_dir, source_format, origin.as_ref())?; + let homepage = match &cli.homepage { Some(h) => { validate_homepage(h)?; @@ -673,7 +816,8 @@ pub async fn resolve(cli: NewCli) -> Result { series, release: cli.release, depends, - native: cli.native, + source_format, + orig, git: cli.git, autopkgtest: false, pkg_config: false, @@ -681,6 +825,111 @@ pub async fn resolve(cli: NewCli) -> Result { }) } +/// Resolve the orig-tarball plan of a quilt scaffold from the CLI answers +/// and the detected git origin. Fails early with a pointed message when an +/// explicit `--orig-from` mode cannot apply to this tree; the implicit, +/// non-interactive default never touches the network (`git archive` of the +/// tag when HEAD sits exactly on one, a working-tree snapshot otherwise). +fn resolve_orig_origin( + cli: &NewCli, + source_dir: &SourceDir, + source_format: SourceFormat, + origin: Option<&GitOrigin>, +) -> Result, String> { + if source_format == SourceFormat::Native { + if cli.orig_from.is_some() || cli.orig_path.is_some() { + return Err( + "--orig-from/--orig-path need a quilt package (drop --native \ + or pass --quilt to build with an orig tarball)" + .to_string(), + ); + } + return Ok(None); + } + + let skeleton = matches!(source_dir, SourceDir::Skeleton); + let head_tag = || origin.and_then(|o| o.head_tag.clone()); + + let plan = match cli.orig_from.as_deref() { + Some("snapshot") => OrigOrigin::Snapshot, + Some("path") => { + let source = cli.orig_path.clone().ok_or_else(|| { + "--orig-from path requires --orig-path ".to_string() + })?; + validate_orig_path(&source)?; + OrigOrigin::Provided { source } + } + Some("git") => { + if skeleton { + return Err("a fresh skeleton has no upstream git history; use \ + --orig-from snapshot (or drop --quilt)" + .to_string()); + } + match head_tag() { + Some(tag) => OrigOrigin::GitArchive { tag }, + None => { + return Err("--orig-from git requires HEAD to be exactly on a \ + release tag" + .to_string()); + } + } + } + Some("release") => { + if skeleton { + return Err("a fresh skeleton has no upstream release to download; \ + use --orig-from snapshot (or drop --quilt)" + .to_string()); + } + let Some(tag) = head_tag() else { + return Err("--orig-from release requires HEAD to be exactly on a \ + release tag" + .to_string()); + }; + let forge = origin.and_then(|o| o.forge.clone()).ok_or_else(|| { + "--orig-from release needs a github.com or gitlab.com origin \ + remote (self-hosted forges are not recognized)" + .to_string() + })?; + OrigOrigin::Release { tag, forge } + } + Some(other) => { + return Err(format!( + "Unknown --orig-from value '{other}'. Supported: release, \ + git, path, snapshot." + )); + } + None => { + // Skeletons (and anything without an upstream history) can only + // be snapshotted; on a tag, git archive is the offline default. + match (skeleton, head_tag()) { + (false, Some(tag)) => OrigOrigin::GitArchive { tag }, + _ => OrigOrigin::Snapshot, + } + } + }; + + if cli.orig_path.is_some() && !matches!(plan, OrigOrigin::Provided { .. }) { + return Err("--orig-path is only valid together with --orig-from path".to_string()); + } + Ok(Some(plan)) +} + +/// Validate a `--orig-path` value (and the wizard's tarball answer): either +/// an http(s) URL (downloaded at scaffold time) or an existing local +/// tarball. +pub fn validate_orig_path(source: &str) -> Result<(), String> { + if source.starts_with("http://") || source.starts_with("https://") { + return Ok(()); + } + if Path::new(source).is_file() { + return Ok(()); + } + Err(format!( + "'{source}' is not an http(s) URL and not an existing file: \ + --orig-path must point at a tarball to repack" + )) +} + /// Set of names no template may produce twice (collision check). pub(crate) fn check_file_collisions(paths: &[String]) -> Result<(), String> { let mut seen: HashSet<&String> = HashSet::new(); @@ -788,7 +1037,8 @@ mod tests { series: "sid".into(), release: false, depends: Vec::new(), - native: false, + source_format: SourceFormat::Quilt, + orig: None, git: false, autopkgtest: false, pkg_config: false, @@ -937,6 +1187,10 @@ mod tests { assert_eq!(opts.depends, vec!["hello (>= 1.0)", "hello-data"]); assert!(opts.git); assert!(!opts.release); + // An existing project (Path mode) is quilt by default; an empty, + // non-git directory has no better orig plan than a tree snapshot. + assert_eq!(opts.source_format, SourceFormat::Quilt); + assert_eq!(opts.orig, Some(OrigOrigin::Snapshot)); // Series: the development series of the current vendor (lowercased, // matching the distro-info keys). let dist = crate::build::env::current_vendor().to_lowercase(); @@ -979,4 +1233,316 @@ mod tests { let err = resolve(cli).await.unwrap_err(); assert!(err.contains("--release requires --series"), "{err}"); } + + /// Minimal CLI answers for a scripted source directory. + fn cli_for(dir: &std::path::Path) -> NewCli { + NewCli { + name: Some("mytool".into()), + source: Some(dir.to_path_buf()), + lang: Some("empty".into()), + description: Some("A tool".into()), + maintainer: Some("Jane ".into()), + git: true, + ..Default::default() + } + } + + /// A git repo fixture: init, commit, optional tag, optional extra + /// commit (needs the git CLI; returns (repo dir, HEAD short hash)). + fn git_repo(tag: Option<&str>, second_commit: bool) -> (tempfile::TempDir, String) { + let run = |dir: &std::path::Path, args: &[&str]| { + let status = std::process::Command::new("git") + .args([ + "-c", + "user.name=T", + "-c", + "user.email=t@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .current_dir(dir) + .status() + .expect("git should be runnable"); + assert!(status.success(), "git {args:?} failed"); + }; + let dir = tempfile::tempdir().unwrap(); + run(dir.path(), &["init", "-q"]); + std::fs::write(dir.path().join("f"), "one\n").unwrap(); + run(dir.path(), &["add", "f"]); + run(dir.path(), &["commit", "-q", "-m", "first"]); + if let Some(tag) = tag { + run(dir.path(), &["tag", tag]); + } + if second_commit { + std::fs::write(dir.path().join("f"), "two\n").unwrap(); + run(dir.path(), &["add", "f"]); + run(dir.path(), &["commit", "-q", "-m", "second"]); + } + let hash = std::process::Command::new("git") + .args(["rev-parse", "--short", "HEAD"]) + .current_dir(dir.path()) + .output() + .unwrap(); + let hash = String::from_utf8_lossy(&hash.stdout).trim().to_string(); + (dir, hash) + } + + #[tokio::test] + async fn resolve_skeleton_is_native_without_orig_tarball() { + // A self-authored skeleton defaults to 3.0 (native): no orig + // tarball, no orig plan. + let cli = NewCli { + name: Some("mytool".into()), + lang: Some("shell".into()), + description: Some("A tool".into()), + maintainer: Some("Jane ".into()), + git: true, + ..Default::default() + }; + let opts = resolve(cli).await.unwrap(); + assert_eq!(opts.source_format, SourceFormat::Native); + assert_eq!(opts.orig, None); + } + + #[tokio::test] + async fn resolve_skeleton_forced_quilt_snapshots_the_tree() { + let cli = NewCli { + name: Some("mytool".into()), + lang: Some("shell".into()), + description: Some("A tool".into()), + maintainer: Some("Jane ".into()), + quilt: true, + git: true, + ..Default::default() + }; + let opts = resolve(cli).await.unwrap(); + assert_eq!(opts.source_format, SourceFormat::Quilt); + assert_eq!(opts.orig, Some(OrigOrigin::Snapshot)); + } + + #[tokio::test] + async fn resolve_skeleton_rejects_history_dependent_orig_modes() { + // A skeleton has no upstream git history: only the snapshot mode + // makes sense. + for mode in ["git", "release"] { + let cli = NewCli { + name: Some("mytool".into()), + lang: Some("shell".into()), + description: Some("A tool".into()), + maintainer: Some("Jane ".into()), + quilt: true, + orig_from: Some(mode.to_string()), + git: true, + ..Default::default() + }; + let err = resolve(cli).await.unwrap_err(); + assert!(err.contains("no upstream"), "{mode}: {err}"); + } + } + + #[tokio::test] + async fn resolve_existing_project_forced_native() { + let dir = tempfile::tempdir().unwrap(); + let mut cli = cli_for(dir.path()); + cli.native = true; + let opts = resolve(cli).await.unwrap(); + assert_eq!(opts.source_format, SourceFormat::Native); + assert_eq!(opts.orig, None); + } + + #[tokio::test] + async fn resolve_native_and_quilt_conflict() { + let cli = NewCli { + name: Some("mytool".into()), + lang: Some("shell".into()), + description: Some("A tool".into()), + maintainer: Some("Jane ".into()), + native: true, + quilt: true, + ..Default::default() + }; + let err = resolve(cli).await.unwrap_err(); + assert!(err.contains("mutually exclusive"), "{err}"); + } + + #[tokio::test] + async fn resolve_head_on_tag_defaults_to_git_archive_and_tag_version() { + if std::process::Command::new("git") + .arg("--version") + .output() + .is_err() + { + return; + } + let (dir, _hash) = git_repo(Some("v1.2.3"), false); + let opts = resolve(cli_for(dir.path())).await.unwrap(); + + assert_eq!(opts.source_format, SourceFormat::Quilt); + // Version default: the tag version, not 0.1.0. + assert_eq!(opts.upstream_version, "1.2.3"); + // Non-interactive orig default on a tag: git archive (no network). + assert_eq!( + opts.orig, + Some(OrigOrigin::GitArchive { + tag: "v1.2.3".to_string() + }) + ); + } + + #[tokio::test] + async fn resolve_between_releases_snapshots_with_git_version() { + if std::process::Command::new("git") + .arg("--version") + .output() + .is_err() + { + return; + } + let (dir, _hash) = git_repo(Some("v1.2.3"), true); + let opts = resolve(cli_for(dir.path())).await.unwrap(); + + let expected = crate::new::origin::GitOrigin::detect(dir.path()) + .unwrap() + .git_version() + .unwrap(); + assert_eq!(opts.upstream_version, expected); + assert!(opts.upstream_version.starts_with("1.2.3+git")); + assert_eq!(opts.orig, Some(OrigOrigin::Snapshot)); + } + + #[tokio::test] + async fn resolve_probed_project_version_beats_the_tag() { + if std::process::Command::new("git") + .arg("--version") + .output() + .is_err() + { + return; + } + let (dir, _) = git_repo(Some("v9.9.9"), false); + // A Cargo.toml without src/ makes `cargo metadata` fail, so the + // template probe falls back to its deterministic line parse. + std::fs::write( + dir.path().join("Cargo.toml"), + "[package]\nname = \"x\"\nversion = \"3.3.3\"\n", + ) + .unwrap(); + let mut cli = cli_for(dir.path()); + cli.lang = Some("rust".into()); + let opts = resolve(cli).await.unwrap(); + assert_eq!(opts.upstream_version, "3.3.3"); + // The orig plan still comes from the tag. + assert_eq!( + opts.orig, + Some(OrigOrigin::GitArchive { + tag: "v9.9.9".to_string() + }) + ); + } + + #[tokio::test] + async fn resolve_orig_from_modes() { + if std::process::Command::new("git") + .arg("--version") + .output() + .is_err() + { + return; + } + let (dir, _) = git_repo(Some("v1.2.3"), false); + + // snapshot is always allowed. + let mut cli = cli_for(dir.path()); + cli.orig_from = Some("snapshot".into()); + assert_eq!(resolve(cli).await.unwrap().orig, Some(OrigOrigin::Snapshot)); + + // release requires a recognized forge remote. + let mut cli = cli_for(dir.path()); + cli.orig_from = Some("release".into()); + let err = resolve(cli).await.unwrap_err(); + assert!(err.contains("github.com or gitlab.com"), "{err}"); + + // After adding a recognized remote the release plan resolves. + std::process::Command::new("git") + .args(["remote", "add", "origin", "https://github.com/foo/bar.git"]) + .current_dir(dir.path()) + .status() + .unwrap() + .success() + .then_some(()) + .unwrap(); + let mut cli = cli_for(dir.path()); + cli.orig_from = Some("release".into()); + assert_eq!( + resolve(cli).await.unwrap().orig, + Some(OrigOrigin::Release { + tag: "v1.2.3".to_string(), + forge: crate::new::origin::Forge::GitHub { + owner: "foo".into(), + repo: "bar".into() + } + }) + ); + + // path requires --orig-path, and an existing file or a URL. + let mut cli = cli_for(dir.path()); + cli.orig_from = Some("path".into()); + let err = resolve(cli).await.unwrap_err(); + assert!(err.contains("--orig-path"), "{err}"); + + let tarball = dir.path().join("upstream.tar.gz"); + std::fs::write(&tarball, b"not really a tarball").unwrap(); + let mut cli = cli_for(dir.path()); + cli.orig_from = Some("path".into()); + cli.orig_path = Some(tarball.to_string_lossy().into_owned()); + assert_eq!( + resolve(cli).await.unwrap().orig, + Some(OrigOrigin::Provided { + source: tarball.to_string_lossy().into_owned() + }) + ); + + let mut cli = cli_for(dir.path()); + cli.orig_from = Some("path".into()); + cli.orig_path = Some("https://example.com/upstream.tar.gz".into()); + assert_eq!( + resolve(cli).await.unwrap().orig, + Some(OrigOrigin::Provided { + source: "https://example.com/upstream.tar.gz".to_string() + }) + ); + + // A missing local path is rejected early. + let mut cli = cli_for(dir.path()); + cli.orig_from = Some("path".into()); + cli.orig_path = Some( + dir.path() + .join("missing.tar.gz") + .to_string_lossy() + .into_owned(), + ); + let err = resolve(cli).await.unwrap_err(); + assert!(err.contains("not an http(s) URL"), "{err}"); + + // --orig-path without --orig-from path is a misuse. + let mut cli = cli_for(dir.path()); + cli.orig_path = Some(tarball.to_string_lossy().into_owned()); + let err = resolve(cli).await.unwrap_err(); + assert!(err.contains("only valid together"), "{err}"); + + // git requires HEAD exactly on a tag. + let (off, _) = git_repo(Some("v1.0.0"), true); + let mut cli = cli_for(off.path()); + cli.orig_from = Some("git".into()); + let err = resolve(cli).await.unwrap_err(); + assert!(err.contains("exactly on a release tag"), "{err}"); + + // ... and the orig flags need quilt. + let mut cli = cli_for(dir.path()); + cli.native = true; + cli.orig_from = Some("git".into()); + let err = resolve(cli).await.unwrap_err(); + assert!(err.contains("need a quilt package"), "{err}"); + } } diff --git a/src/new/orig.rs b/src/new/orig.rs new file mode 100644 index 0000000..77da61e --- /dev/null +++ b/src/new/orig.rs @@ -0,0 +1,1103 @@ +//! Orig tarball creation for `pkh new`, one implementation per +//! [`OrigOrigin`]: working-tree snapshot, `git archive` of a release tag, +//! download of the forge release tarball, or repack of a user-provided +//! tarball — plus the dpkg upstream component tarball holding the vendored +//! Cargo dependencies (`_.orig-vendor.tar.xz`). +//! +//! Whatever the origin, the tarball always lands at +//! `../_.orig.tar.xz` with `-/` as its single +//! top-level directory (what dpkg-source expects), and — for a vendored +//! rust package — always excludes the generated `vendor/` tree, which +//! travels in the component tarball instead and can be regenerated +//! independently of the upstream sources. + +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use xz2::write::XzEncoder; + +use super::options::OrigOrigin; +use crate::debian::DebianVersion; +use crate::new::origin::Forge; + +/// The upstream version dpkg names orig/component files after: the +/// changelog version's upstream part, with the epoch and the Debian +/// revision stripped (`1:0.14.0-1` → `0.14.0`). +/// +/// dpkg-source globs `../_.orig.tar.xz` and +/// `../_.orig-.tar.` — never the full +/// version — so every lookup or creation of those artifacts must derive +/// the name through here. Using `DebianVersion::no_epoch()` instead yields +/// `0.14.0-1` and silently misses the real component. +pub fn component_upstream_version(changelog_version: &DebianVersion) -> &str { + &changelog_version.upstream +} + +/// Result of the orig creation: where the tarball landed and how it was +/// actually produced (the release download falls through to `git archive` +/// or a snapshot on failure, so the label can differ from the plan). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreatedOrig { + /// Path of the written `_.orig.tar.xz`. + pub path: PathBuf, + /// Human-readable description of the actual origin. + pub label: String, +} + +/// Create the orig tarball of a quilt package according to `plan`. +/// +/// `vendored_rust` marks a rust package whose tree carries a generated +/// `vendor/` directory: the snapshot origin then excludes it (the other +/// origins never contain it in the first place — upstream tags predate the +/// vendoring). Failures of an explicit user choice (`--orig-from path`) +/// are fatal; a failed release download is only a warning and falls +/// through to `git archive` of the tag, then to the snapshot. +pub fn create_orig( + tree: &Path, + name: &str, + upstream_version: &str, + plan: &OrigOrigin, + vendored_rust: bool, +) -> Result> { + match plan { + OrigOrigin::Snapshot => { + let path = super::debian::create_orig_tarball_excluding( + tree, + name, + upstream_version, + vendored_rust, + )?; + Ok(CreatedOrig { + path, + label: OrigOrigin::Snapshot.label(), + }) + } + OrigOrigin::GitArchive { tag } => { + let path = git_archive_tarball(tree, tag, name, upstream_version)?; + Ok(CreatedOrig { + path, + label: OrigOrigin::GitArchive { tag: tag.clone() }.label(), + }) + } + OrigOrigin::Release { tag, forge } => { + match download_release(forge, tag, name, upstream_version, tree) { + Ok(path) => Ok(CreatedOrig { + path, + label: OrigOrigin::Release { + tag: tag.clone(), + forge: forge.clone(), + } + .label(), + }), + Err(download_error) => { + log::warn!( + "The release tarball of {tag} could not be downloaded \ + ({download_error:#}); falling back to `git archive` \ + of the tag, then to a working-tree snapshot" + ); + match git_archive_tarball(tree, tag, name, upstream_version) { + Ok(path) => Ok(CreatedOrig { + path, + label: OrigOrigin::GitArchive { tag: tag.clone() }.label(), + }), + Err(archive_error) => { + log::warn!( + "`git archive` of {tag} failed too ({archive_error:#}); \ + snapshotting the working tree instead" + ); + let path = super::debian::create_orig_tarball_excluding( + tree, + name, + upstream_version, + vendored_rust, + )?; + Ok(CreatedOrig { + path, + label: OrigOrigin::Snapshot.label(), + }) + } + } + } + } + } + OrigOrigin::Provided { source } => { + let path = fetch_and_repack(source, name, upstream_version, tree)?; + Ok(CreatedOrig { + path, + label: OrigOrigin::Provided { + source: source.clone(), + } + .label(), + }) + } + } +} + +/// Create `../_.orig-vendor.tar.xz` holding the tree's +/// `vendor/` directory under a top-level `vendor/` path — the dpkg upstream +/// component dpkg-source unpacks back into the tree next to the main orig. +/// Refuses to overwrite an existing component (stale components are +/// removed by the caller, e.g. the re-vendoring retry of `pkh build`). +pub fn create_vendor_component( + tree: &Path, + name: &str, + upstream_version: &str, +) -> Result> { + let vendor = tree.join("vendor"); + if !is_non_empty_dir(&vendor) { + return Err(format!( + "'{}' does not exist or is empty: nothing to put into the \ + orig-vendor component", + vendor.display() + ) + .into()); + } + let component_path = vendor_component_path(tree, name, upstream_version).ok_or_else(|| { + format!( + "cannot determine the parent directory of '{}'", + tree.display() + ) + })?; + if component_path.exists() { + return Err(format!( + "'{}' already exists: pkh new refuses to overwrite it. \ + Remove the stale component first.", + component_path.display() + ) + .into()); + } + + let file = std::fs::File::create(&component_path)?; + let encoder = XzEncoder::new(file, 6); + let mut builder = tar::Builder::new(encoder); + // Everything under vendor/ travels; no exclusions (vendored crates have + // no build leftovers) and no debian/ special case below the top level. + builder.append_dir("vendor", &vendor)?; + super::debian::append_tree(&mut builder, &vendor, "vendor", 0, &[], &[])?; + builder + .finish() + .map_err(|e| format!("failed to write '{}': {}", component_path.display(), e))?; + + log::info!( + "Created vendored-dependencies component {}", + crate::ui::display_path(&component_path) + ); + Ok(component_path) +} + +/// Path of the dpkg upstream component holding `vendor/`, next to the tree. +pub fn vendor_component_path(tree: &Path, name: &str, upstream_version: &str) -> Option { + tree.parent() + .map(|parent| parent.join(format!("{name}_{upstream_version}.orig-vendor.tar.xz"))) +} + +/// Whether `tree` carries a generated, non-empty `vendor/` directory. +pub fn has_vendored_dir(tree: &Path) -> bool { + is_non_empty_dir(&tree.join("vendor")) +} + +fn is_non_empty_dir(path: &Path) -> bool { + path.is_dir() && std::fs::read_dir(path).is_ok_and(|mut entries| entries.next().is_some()) +} + +/// `git archive --format=tar --prefix=-/ ` compressed to +/// `../_.orig.tar.xz`: offline and byte-deterministic. +fn git_archive_tarball( + repo: &Path, + tag: &str, + name: &str, + upstream_version: &str, +) -> Result> { + let dest = super::debian::orig_tarball_path(repo, name, upstream_version).ok_or_else(|| { + format!( + "cannot determine the parent directory of '{}'", + repo.display() + ) + })?; + if dest.exists() { + return Err(format!( + "'{}' already exists: pkh new refuses to overwrite it.", + dest.display() + ) + .into()); + } + + let file = std::fs::File::create(&dest)?; + let mut encoder = XzEncoder::new(file, 6); + let mut child = Command::new("git") + .args([ + "archive", + "--format=tar", + &format!("--prefix={name}-{upstream_version}/"), + tag, + ]) + .current_dir(repo) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("failed to run 'git archive': {e}"))?; + let mut stdout = child + .stdout + .take() + .ok_or_else(|| "git archive produced no output".to_string())?; + std::io::copy(&mut stdout, &mut encoder) + .map_err(|e| format!("cannot pipe git archive into '{}': {e}", dest.display()))?; + encoder + .finish() + .map_err(|e| format!("failed to write '{}': {e}", dest.display()))?; + let status = child.wait()?; + if !status.success() { + // The half-written xz file is not a valid tarball: remove it. + let _ = std::fs::remove_file(&dest); + return Err(format!( + "'git archive --format=tar {}' failed with status: {status} \ + (is HEAD exactly on the tag '{tag}'?)", + tag + ) + .into()); + } + + log::info!( + "Created orig tarball from git archive of {tag}: {}", + crate::ui::display_path(&dest) + ); + Ok(dest) +} + +/// Download the release tarball of `tag` from `forge` (the user's choice of +/// this origin IS the network consent) and repack it to +/// `../_.orig.tar.xz`. All of the forge's candidate URLs are +/// tried before failing. +fn download_release( + forge: &Forge, + tag: &str, + name: &str, + upstream_version: &str, + tree: &Path, +) -> Result> { + let urls = forge.release_tarball_urls(tag); + let mut last_error: Option> = None; + for url in &urls { + log::info!("Downloading the upstream release tarball from {url}"); + match download_to_temp(url) { + Ok(temp) => { + let result = repack_tarball_file(&temp, name, upstream_version, tree); + let _ = std::fs::remove_file(&temp); + return match result { + Ok(path) => { + log::info!( + "Created orig tarball from the release download of {tag}: {}", + crate::ui::display_path(&path) + ); + Ok(path) + } + Err(e) => Err(format!( + "the downloaded tarball of {url} is not a \ + usable tar archive: {e}" + ) + .into()), + }; + } + Err(error) => { + log::warn!("Download from {url} failed: {error}"); + last_error = Some(error); + } + } + } + Err(last_error.unwrap_or_else(|| format!("no download URL known for {forge:?}").into())) +} + +/// Repack a user-provided tarball (a local path or an http(s) URL, +/// `.tar`/`.tar.gz`/`.tgz`/`.tar.bz2`/`.tbz2`/`.tar.xz`) into +/// `../_.orig.tar.xz`. +fn fetch_and_repack( + source: &str, + name: &str, + upstream_version: &str, + tree: &Path, +) -> Result> { + let temp; + let path: &Path = if source.starts_with("http://") || source.starts_with("https://") { + log::info!("Downloading the user-provided tarball from {source}"); + temp = download_to_temp(source)?; + &temp + } else { + Path::new(source) + }; + let dest = repack_tarball_file(path, name, upstream_version, tree)?; + log::info!( + "Created orig tarball from {}: {}", + source, + crate::ui::display_path(&dest) + ); + Ok(dest) +} + +/// Download `url` into a fresh temporary file, returning its path. The +/// blocking client must not run on a tokio worker thread (scaffold is +/// called from inside the async runtime), so the download runs on a plain +/// dedicated thread. +fn download_to_temp(url: &str) -> Result> { + let url = url.to_string(); + let contents = std::thread::spawn(move || { + reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(30)) + .build()? + .get(&url) + .send()? + .error_for_status()? + .bytes() + }) + .join() + .map_err(|_| -> Box { "the download thread panicked".into() })??; + + let temp = std::env::temp_dir().join(format!( + "pkh-orig-{}-{}", + std::process::id(), + chrono::Utc::now().timestamp_nanos_opt().unwrap_or_default() + )); + let mut file = std::fs::File::create(&temp)?; + file.write_all(&contents)?; + Ok(temp) +} + +/// Open `path` as a (possibly compressed) tar stream and repack it under +/// the `-/` prefix into `../_.orig.tar.xz`. +/// `.tar.bz2`/`.tbz2` inputs are decompressed through the host `bzip2` +/// binary (pkh carries no bzip2 codec); gz and xz are decoded natively. +fn repack_tarball_file( + path: &Path, + name: &str, + upstream_version: &str, + tree: &Path, +) -> Result> { + let dest = super::debian::orig_tarball_path(tree, name, upstream_version).ok_or_else(|| { + format!( + "cannot determine the parent directory of '{}'", + tree.display() + ) + })?; + if dest.exists() { + return Err(format!( + "'{}' already exists: pkh new refuses to overwrite it.", + dest.display() + ) + .into()); + } + + let extension = path + .extension() + .and_then(|ext| ext.to_str()) + .unwrap_or_default() + .to_ascii_lowercase(); + let full = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default() + .to_ascii_lowercase(); + + let mut bzip2_child: Option = None; + let reader: Box = if full.ends_with(".tar.gz") || full.ends_with(".tgz") { + Box::new(flate2::read::GzDecoder::new(std::fs::File::open(path)?)) + } else if full.ends_with(".tar.xz") || full.ends_with(".txz") { + Box::new(xz2::read::XzDecoder::new(std::fs::File::open(path)?)) + } else if full.ends_with(".tar.bz2") || full.ends_with(".tbz2") { + let mut child = Command::new("bzip2") + .arg("-dc") + .arg(path) + .stdout(Stdio::piped()) + .spawn() + .map_err(|e| { + format!( + "'.tar.bz2' tarballs need the bzip2 binary on PATH to be \ + repacked: {e}" + ) + })?; + let stdout = child + .stdout + .take() + .ok_or_else(|| "bzip2 produced no output".to_string())?; + bzip2_child = Some(child); + Box::new(stdout) + } else if extension == "tar" { + Box::new(std::fs::File::open(path)?) + } else { + return Err(format!( + "'{}' does not look like a tarball: expected .tar, .tar.gz, \ + .tgz, .tar.bz2, .tbz2 or .tar.xz", + path.display() + ) + .into()); + }; + + let result = repack_tar_stream(reader, name, upstream_version, &dest); + if let Some(mut child) = bzip2_child + && let Ok(dest_path) = &result + { + let status = child.wait()?; + if !status.success() { + let _ = std::fs::remove_file(dest_path); + return Err(format!( + "'bzip2 -dc {}' failed with status: {status}", + path.display() + ) + .into()); + } + } + if result.is_err() { + // A failed repack must not leave a half-written tarball behind. + let _ = std::fs::remove_file(&dest); + } + result +} + +/// Rewrite every entry of the tar `stream` under the `-/` +/// top-level directory (whatever prefix the source tarball used) into the +/// xz-compressed tarball at `dest`. `.git` directories and tar metadata +/// leftovers are dropped, modes travel through. +fn repack_tar_stream( + stream: Box, + name: &str, + upstream_version: &str, + dest: &Path, +) -> Result> { + let file = std::fs::File::create(dest)?; + let encoder = XzEncoder::new(file, 6); + let mut builder = tar::Builder::new(encoder); + let prefix = format!("{name}-{upstream_version}"); + + let mut archive = tar::Archive::new(stream); + for entry in archive.entries()? { + let mut entry = entry?; + let original = entry.path()?.to_path_buf(); + // Drop the source tarball's top-level directory... + let rest: PathBuf = original + .components() + .skip(1) + .filter(|component| component.as_os_str() != ".git") + .collect(); + // ...skipping the top-level entry itself and any entry living + // inside a dropped directory (empty `rest` after a `.git` strip). + if rest.as_os_str().is_empty() { + continue; + } + if original + .file_name() + .is_some_and(|name| name == "pax_global_header") + { + continue; + } + let new_path = format!("{prefix}/{}", rest.to_string_lossy()); + + let mut header = entry.header().clone(); + match header.entry_type() { + tar::EntryType::Directory => { + builder.append_data(&mut header, &new_path, std::io::empty())?; + } + tar::EntryType::Regular | tar::EntryType::Continuous => { + builder.append_data(&mut header, &new_path, &mut entry)?; + } + tar::EntryType::Symlink | tar::EntryType::Link => { + let target = entry + .link_name()? + .ok_or_else(|| format!("'{original:?}' is a link without a target"))?; + builder.append_link(&mut header, &new_path, target)?; + } + other => { + log::warn!( + "Skipping {other:?} entry '{original:?}' while repacking \ + the orig tarball" + ); + } + } + } + + builder + .finish() + .map_err(|e| format!("failed to write '{}': {e}", dest.display()))?; + Ok(dest.to_path_buf()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// List the entry names of an xz tarball. + fn tarball_names(path: &Path) -> Vec { + let mut archive = tar::Archive::new(xz2::read::XzDecoder::new( + std::fs::File::open(path).unwrap(), + )); + archive + .entries() + .unwrap() + .map(|entry| { + entry + .unwrap() + .path() + .unwrap() + .to_string_lossy() + .into_owned() + }) + .collect() + } + + /// Build a gz tarball with the given entries (path → contents), under + /// the `oldpkg-1.0/` top-level directory. + fn write_gz_fixture(path: &Path, entries: &[(&str, &str)]) { + let file = std::fs::File::create(path).unwrap(); + let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast()); + let mut builder = tar::Builder::new(encoder); + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Directory); + header.set_size(0); + header.set_mode(0o755); + header.set_cksum(); + builder + .append_data(&mut header, "oldpkg-1.0", std::io::empty()) + .unwrap(); + for (name, contents) in entries { + let mut header = tar::Header::new_gnu(); + header.set_size(contents.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data( + &mut header, + format!("oldpkg-1.0/{name}"), + contents.as_bytes(), + ) + .unwrap(); + } + builder.into_inner().unwrap(); + } + + #[test] + fn repack_rewrites_the_top_level_prefix() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(&tree).unwrap(); + + let source = dir.path().join("old.tar.gz"); + write_gz_fixture(&source, &[("src/main.rs", "hi\n"), ("README", "readme\n")]); + + let dest = repack_tarball_file(&source, "mytool", "1.2.3", &tree).unwrap(); + assert_eq!(dest, dir.path().join("mytool_1.2.3.orig.tar.xz")); + let names = tarball_names(&dest); + assert!( + names.iter().any(|n| n == "mytool-1.2.3/src/main.rs"), + "{names:?}" + ); + assert!( + names.iter().any(|n| n == "mytool-1.2.3/README"), + "{names:?}" + ); + assert!( + !names.iter().any(|n| n.starts_with("oldpkg-1.0")), + "{names:?}" + ); + } + + #[test] + fn repack_drops_git_dirs_and_unsupported_entries() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(&tree).unwrap(); + + // A gz tarball carrying a .git directory next to real sources. + let source = dir.path().join("old.tar.gz"); + write_gz_fixture( + &source, + &[(".git/config", "ignored"), ("src/lib.rs", "code")], + ); + + let dest = repack_tarball_file(&source, "mytool", "0.1.0", &tree).unwrap(); + let names = tarball_names(&dest); + assert!( + names.iter().any(|n| n == "mytool-0.1.0/src/lib.rs"), + "{names:?}" + ); + assert!(!names.iter().any(|n| n.contains(".git")), "{names:?}"); + } + + #[test] + fn repack_rejects_unknown_extensions_and_existing_dest() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(&tree).unwrap(); + + let source = dir.path().join("upstream.tar.zip"); + std::fs::write(&source, b"zip").unwrap(); + let err = repack_tarball_file(&source, "mytool", "0.1.0", &tree).unwrap_err(); + assert!( + err.to_string().contains("does not look like a tarball"), + "{err}" + ); + + // An existing destination is refused before anything is unpacked. + let source = dir.path().join("upstream.tar"); + std::fs::write(&source, b"").unwrap(); + std::fs::write(dir.path().join("mytool_0.1.0.orig.tar.xz"), b"old").unwrap(); + let err = repack_tarball_file(&source, "mytool", "0.1.0", &tree).unwrap_err(); + assert!(err.to_string().contains("already exists"), "{err}"); + } + + #[test] + fn repack_supports_plain_and_xz_tarballs() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(&tree).unwrap(); + + // Plain .tar. + let source = dir.path().join("upstream.tar"); + { + let mut builder = tar::Builder::new(std::fs::File::create(&source).unwrap()); + let mut header = tar::Header::new_gnu(); + header.set_size(3); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "oldpkg-1.0/f.txt", "abc".as_bytes()) + .unwrap(); + builder.finish().unwrap(); + } + let dest = repack_tarball_file(&source, "mytool", "2.0", &tree).unwrap(); + assert!(tarball_names(&dest).contains(&"mytool-2.0/f.txt".to_string())); + + // .tar.xz. + let source = dir.path().join("upstream.tar.xz"); + { + let file = std::fs::File::create(&source).unwrap(); + let encoder = xz2::write::XzEncoder::new(file, 1); + let mut builder = tar::Builder::new(encoder); + let mut header = tar::Header::new_gnu(); + header.set_size(3); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "oldpkg-1.0/g.txt", "xyz".as_bytes()) + .unwrap(); + builder.finish().unwrap(); + } + let dest = repack_tarball_file(&source, "mytool", "2.1", &tree).unwrap(); + assert!(tarball_names(&dest).contains(&"mytool-2.1/g.txt".to_string())); + } + + #[test] + fn repack_preserves_the_exec_bit() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(&tree).unwrap(); + + let source = dir.path().join("upstream.tar"); + { + let mut builder = tar::Builder::new(std::fs::File::create(&source).unwrap()); + let mut header = tar::Header::new_gnu(); + header.set_size(11); + header.set_mode(0o755); + header.set_cksum(); + builder + .append_data( + &mut header, + "oldpkg-1.0/run.sh", + "#!/bin/sh\nx\n".as_bytes(), + ) + .unwrap(); + builder.finish().unwrap(); + } + let dest = repack_tarball_file(&source, "mytool", "0.5.0", &tree).unwrap(); + let mut archive = tar::Archive::new(xz2::read::XzDecoder::new( + std::fs::File::open(&dest).unwrap(), + )); + for entry in archive.entries().unwrap() { + let entry = entry.unwrap(); + if entry.path().unwrap().ends_with("run.sh") { + assert_eq!(entry.header().mode().unwrap() & 0o111, 0o111); + } + } + } + + #[test] + fn vendor_component_layout_and_overwrite_refusal() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(tree.join("vendor/serde/src")).unwrap(); + std::fs::write(tree.join("vendor/serde/src/lib.rs"), "crate code").unwrap(); + std::fs::write(tree.join("vendor/serde/Cargo.toml"), "[package]").unwrap(); + + let component = create_vendor_component(&tree, "mytool", "1.0.0").unwrap(); + assert_eq!( + component, + dir.path().join("mytool_1.0.0.orig-vendor.tar.xz") + ); + let names = tarball_names(&component); + assert!( + names.iter().any(|n| n == "vendor/serde/src/lib.rs"), + "{names:?}" + ); + // The top-level entry is the bare `vendor/` directory. + assert!( + names.iter().any(|n| n.trim_end_matches('/') == "vendor"), + "{names:?}" + ); + + // A second creation refuses to overwrite the stale component. + let err = create_vendor_component(&tree, "mytool", "1.0.0").unwrap_err(); + assert!(err.to_string().contains("already exists"), "{err}"); + } + + #[test] + fn vendor_component_needs_a_non_empty_vendor_dir() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(&tree).unwrap(); + // No vendor/ at all. + let err = create_vendor_component(&tree, "mytool", "1.0.0").unwrap_err(); + assert!(err.to_string().contains("vendor"), "{err}"); + // An empty vendor/ counts as nothing. + std::fs::create_dir_all(tree.join("vendor")).unwrap(); + assert!(create_vendor_component(&tree, "mytool", "1.0.0").is_err()); + } + + /// Regression (pkh build re-vendor retry): the component file name is + /// derived from the changelog version's UPSTREAM part, never the full + /// version. The retry hook must therefore compute + /// `_0.14.0.orig-vendor.tar.xz` for changelog version + /// `0.14.0-1` — the exact file `create_vendor_component` names — or a + /// stale component silently survives the recreation. + #[test] + fn component_name_uses_the_upstream_version_part() { + use crate::debian::DebianVersion; + + // `0.14.0-1`: upstream part only. NOT `0.14.0-1` + // (`DebianVersion::no_epoch()`), which was the original bug. + let version = DebianVersion::parse("0.14.0-1").unwrap(); + assert_eq!(component_upstream_version(&version), "0.14.0"); + // Epochs are stripped the same way. + let epochy = DebianVersion::parse("2:0.14.0-1").unwrap(); + assert_eq!(component_upstream_version(&epochy), "0.14.0"); + + // End-to-end naming: the path the retry hook looks up and the file + // `create_vendor_component` writes are one and the same. + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(tree.join("vendor/serde")).unwrap(); + std::fs::write(tree.join("vendor/serde/lib.rs"), "code").unwrap(); + let created = + create_vendor_component(&tree, "mytool", component_upstream_version(&version)).unwrap(); + assert_eq!( + created, + vendor_component_path(&tree, "mytool", component_upstream_version(&version)).unwrap() + ); + assert_eq!( + created.file_name().unwrap(), + std::ffi::OsStr::new("mytool_0.14.0.orig-vendor.tar.xz") + ); + // No revision-suffixed variant may exist next to it. + assert!( + !dir.path() + .join("mytool_0.14.0-1.orig-vendor.tar.xz") + .exists() + ); + } + + #[test] + fn has_vendored_dir_detection() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(&tree).unwrap(); + assert!(!has_vendored_dir(&tree)); + std::fs::create_dir_all(tree.join("vendor")).unwrap(); + assert!(!has_vendored_dir(&tree)); // empty + std::fs::write(tree.join("vendor/x"), "y").unwrap(); + assert!(has_vendored_dir(&tree)); + } + + /// The snapshot origin with `vendored_rust` excludes the top-level + /// `vendor/` from the main orig (that is what the component is for). + #[test] + fn snapshot_origin_excludes_vendor_when_vendored() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(tree.join("vendor/serde/src")).unwrap(); + std::fs::write(tree.join("vendor/serde/src/lib.rs"), "code").unwrap(); + std::fs::write(tree.join("Cargo.toml"), "[package]").unwrap(); + + let created = create_orig(&tree, "mytool", "1.0.0", &OrigOrigin::Snapshot, true).unwrap(); + assert_eq!(created.label, "working tree snapshot"); + let names = tarball_names(&created.path); + assert!( + names.iter().any(|n| n == "mytool-1.0.0/Cargo.toml"), + "{names:?}" + ); + assert!(!names.iter().any(|n| n.contains("vendor")), "{names:?}"); + + // Without the vendored-rust marker the directory stays in (a + // non-rust project may legitimately carry one). + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(tree.join("src/vendor")).unwrap(); + std::fs::write(tree.join("src/vendor/mod.rs"), "code").unwrap(); + let created = create_orig(&tree, "mytool", "1.0.0", &OrigOrigin::Snapshot, false).unwrap(); + let names = tarball_names(&created.path); + assert!( + names.iter().any(|n| n == "mytool-1.0.0/src/vendor/mod.rs"), + "{names:?}" + ); + } + + /// git archive origin: a scripted repo with a tag produces exactly the + /// tagged content under the `-/` prefix. + #[test] + fn git_archive_origin_packs_the_tag_content() { + if std::process::Command::new("git") + .arg("--version") + .output() + .is_err() + { + return; + } + let dir = tempfile::tempdir().unwrap(); + let repo = dir.path().join("mytool"); + std::fs::create_dir_all(&repo).unwrap(); + let git = |args: &[&str]| { + let status = std::process::Command::new("git") + .args([ + "-c", + "user.name=T", + "-c", + "user.email=t@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .current_dir(&repo) + .status() + .unwrap(); + assert!(status.success(), "git {args:?} failed"); + }; + git(&["init", "-q"]); + std::fs::write(repo.join("hello.txt"), "release\n").unwrap(); + git(&["add", "hello.txt"]); + git(&["commit", "-q", "-m", "release"]); + git(&["tag", "v1.2.3"]); + // A later, uncommitted-looking file exists in the worktree but must + // NOT travel into the tag archive. + std::fs::write(repo.join("uncommitted.txt"), "dirty\n").unwrap(); + + let created = create_orig( + &repo, + "mytool", + "1.2.3", + &OrigOrigin::GitArchive { + tag: "v1.2.3".to_string(), + }, + false, + ) + .unwrap(); + assert_eq!(created.label, "git archive (v1.2.3)"); + let names = tarball_names(&created.path); + assert!( + names.iter().any(|n| n == "mytool-1.2.3/hello.txt"), + "{names:?}" + ); + assert!( + !names.iter().any(|n| n.contains("uncommitted")), + "{names:?}" + ); + } + + /// A failing git archive (tag missing) is reported, not silently + /// swallowed. + #[test] + fn git_archive_origin_fails_on_a_missing_tag() { + if std::process::Command::new("git") + .arg("--version") + .output() + .is_err() + { + return; + } + let dir = tempfile::tempdir().unwrap(); + let repo = dir.path().join("mytool"); + std::fs::create_dir_all(&repo).unwrap(); + let status = std::process::Command::new("git") + .args(["init", "-q"]) + .current_dir(&repo) + .status() + .unwrap(); + assert!(status.success()); + + let err = create_orig( + &repo, + "mytool", + "1.0.0", + &OrigOrigin::GitArchive { + tag: "v9.9.9".to_string(), + }, + false, + ) + .unwrap_err(); + assert!(err.to_string().contains("git archive"), "{err}"); + // No half-written tarball is left behind. + assert!(!dir.path().join("mytool_1.0.0.orig.tar.xz").exists()); + } + + /// The release origin falls through to `git archive` when the download + /// fails (offline host): the label reflects the actual origin. + #[test] + fn release_origin_falls_through_to_git_archive() { + if std::process::Command::new("git") + .arg("--version") + .output() + .is_err() + { + return; + } + let dir = tempfile::tempdir().unwrap(); + let repo = dir.path().join("mytool"); + std::fs::create_dir_all(&repo).unwrap(); + let git = |args: &[&str]| { + let status = std::process::Command::new("git") + .args([ + "-c", + "user.name=T", + "-c", + "user.email=t@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .current_dir(&repo) + .status() + .unwrap(); + assert!(status.success(), "git {args:?} failed"); + }; + git(&["init", "-q"]); + std::fs::write(repo.join("f"), "x\n").unwrap(); + git(&["add", "f"]); + git(&["commit", "-q", "-m", "first"]); + git(&["tag", "v0.1.0"]); + + // An unreachable URL: the download fails quickly (connection + // refused on a reserved port), the git archive takes over. + let created = create_orig( + &repo, + "mytool", + "0.1.0", + &OrigOrigin::Release { + tag: "v0.1.0".to_string(), + forge: Forge::GitHub { + owner: "pkh-nonexistent-org".into(), + repo: "pkh-nonexistent-repo".into(), + }, + }, + false, + ) + .unwrap(); + assert_eq!(created.label, "git archive (v0.1.0)"); + assert!(created.path.exists()); + } + + /// The provided origin accepts a local gz tarball and repacks it. + #[test] + fn provided_origin_repacks_a_local_tarball() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(&tree).unwrap(); + + let source = dir.path().join("given.tar.gz"); + write_gz_fixture(&source, &[("main.rs", "fn main() {}\n")]); + + let created = create_orig( + &tree, + "mytool", + "3.0.0", + &OrigOrigin::Provided { + source: source.to_string_lossy().into_owned(), + }, + false, + ) + .unwrap(); + assert!(created.label.starts_with("user tarball")); + let names = tarball_names(&created.path); + assert!( + names.iter().any(|n| n == "mytool-3.0.0/main.rs"), + "{names:?}" + ); + } + + /// Sanity: a tar builder accepts empty writes for directory headers + /// (the repack path relies on it). + #[test] + fn empty_write_directory_header_roundtrip() { + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path().join("t.tar"); + let mut builder = tar::Builder::new(std::fs::File::create(&dest).unwrap()); + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Directory); + header.set_size(0); + header.set_mode(0o755); + header.set_cksum(); + builder + .append_data(&mut header, "top", std::io::empty()) + .unwrap(); + builder.finish().unwrap(); + let mut archive = tar::Archive::new(std::fs::File::open(&dest).unwrap()); + let names: Vec = archive + .entries() + .unwrap() + .map(|e| e.unwrap().path().unwrap().to_string_lossy().into_owned()) + .collect(); + assert_eq!(names, vec!["top"]); + } + + /// Ensure writes into the gz fixture builder produce a readable tarball + /// (trips on header size mismatches). + #[test] + fn gz_fixture_builder_produces_readable_tarballs() { + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("fixture.tar.gz"); + write_gz_fixture(&source, &[("a.txt", "A")]); + let decoder = flate2::read::GzDecoder::new(std::fs::File::open(&source).unwrap()); + let mut archive = tar::Archive::new(decoder); + let mut seen = Vec::new(); + for entry in archive.entries().unwrap() { + let mut entry = entry.unwrap(); + let name = entry.path().unwrap().to_string_lossy().into_owned(); + seen.push(name.clone()); + if name.ends_with('/') || entry.header().entry_type() == tar::EntryType::Directory { + continue; + } + let mut contents = String::new(); + entry.read_to_string(&mut contents).unwrap(); + assert_eq!(contents, "A"); + } + assert!(seen.contains(&"oldpkg-1.0/a.txt".to_string()), "{seen:?}"); + } + + /// Ensure writes into the xz encoder fail loudly when the stream is + /// not a tar at all (garbage input produces a readable error path). + #[test] + fn repack_garbage_fails_cleanly() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(&tree).unwrap(); + let source = dir.path().join("garbage.tar.gz"); + { + let mut file = std::fs::File::create(&source).unwrap(); + file.write_all(b"definitely not gzip").unwrap(); + } + let result = repack_tarball_file(&source, "mytool", "1.0.0", &tree); + // Either the gzip header check fails immediately or no tarball is + // left over — both are acceptable failures, a corrupt orig is not. + if let Ok(dest) = result { + assert!( + tarball_names(&dest).is_empty() || !dest.exists(), + "garbage input must not produce a usable orig" + ); + } + assert!(!dir.path().join("mytool_1.0.0.orig.tar.xz").exists()); + } +} diff --git a/src/new/origin.rs b/src/new/origin.rs new file mode 100644 index 0000000..4930eef --- /dev/null +++ b/src/new/origin.rs @@ -0,0 +1,533 @@ +//! Best-effort git origin detection for `pkh new`. +//! +//! [`GitOrigin::detect`] inspects the source directory through the `git` +//! CLI (fail-soft: any failed query just leaves the corresponding field +//! empty, and a non-repo yields `None`) and answers the questions driving +//! the source-format and orig-tarball decisions: +//! +//! - is HEAD exactly on a tag, and which upstream version does it name, +//! - which is the last tag reachable from HEAD (for the +//! `+git.` version scheme), +//! - where does the `origin` remote point: only `github.com` and +//! `gitlab.com` are recognized as forges — self-hosted GitLab instances +//! are deliberately not (the release-download URL shapes differ). +//! +//! Detection never touches the network: adding a remote stores its URL in +//! the local config only, which is all this module reads. + +use std::path::Path; +use std::process::Command; + +/// A forge hosting the project, parsed from the `origin` remote URL. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Forge { + /// `github.com//` + GitHub { + /// Repository owner (user or organization). + owner: String, + /// Repository name, without the `.git` suffix. + repo: String, + }, + /// `gitlab.com//` + GitLab { + /// Repository owner (user or group). + owner: String, + /// Repository name, without the `.git` suffix. + repo: String, + }, +} + +impl Forge { + /// Host name of the forge. + pub fn host(&self) -> &'static str { + match self { + Forge::GitHub { .. } => "github.com", + Forge::GitLab { .. } => "gitlab.com", + } + } + + /// Parse a remote URL into a [`Forge`], accepting the `https://`, + /// `http://`, `git://` and `git@host:` spellings. Only `github.com` and + /// `gitlab.com` are recognized; anything else (self-hosted GitLab, + /// Bitbucket, plain URLs…) yields `None`. + pub fn parse(url: &str) -> Option { + let url = url.trim(); + // Normalize `git@host:path` to `host/path` and strip any scheme. + let (host, path) = match url.split_once("://") { + Some((_scheme, rest)) => rest.split_once('/')?, + None => { + let (scp, path) = url.split_once(':')?; + let host = scp.strip_prefix("git@").unwrap_or(scp); + (host, path) + } + }; + let host = host.to_ascii_lowercase(); + // Drop the leading user part of ssh URLs (`git@github.com` handled + // above; `ssh://git@github.com/path` keeps `git@github.com` here). + let host = host.rsplit('@').next().unwrap_or(&host); + let mut segments = path + .trim_end_matches('/') + .trim_end_matches(".git") + .split('/'); + let owner = segments.next()?; + let repo = segments.next()?; + if owner.is_empty() || repo.is_empty() { + return None; + } + match host { + "github.com" => Some(Forge::GitHub { + owner: owner.to_string(), + repo: repo.to_string(), + }), + "gitlab.com" => Some(Forge::GitLab { + owner: owner.to_string(), + repo: repo.to_string(), + }), + _ => None, + } + } + + /// Release-tarball URLs of `tag`, best candidate first. GitHub prefers + /// the codeload direct link (no redirect) and falls back to the + /// `github.com` archive URL; GitLab has a single archive URL. + pub fn release_tarball_urls(&self, tag: &str) -> Vec { + match self { + Forge::GitHub { owner, repo } => vec![ + format!("https://codeload.github.com/{owner}/{repo}/tar.gz/refs/tags/{tag}"), + format!("https://github.com/{owner}/{repo}/archive/refs/tags/{tag}.tar.gz"), + ], + Forge::GitLab { owner, repo } => { + vec![format!( + "https://gitlab.com/{owner}/{repo}/-/archive/{tag}/{repo}-{tag}.tar.gz" + )] + } + } + } +} + +/// Best-effort snapshot of the git state of a source directory (see the +/// module docs). Every field degrades to its empty value when the +/// corresponding query fails. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct GitOrigin { + /// Forge of the `origin` remote, when it is one of the recognized ones. + pub forge: Option, + /// Tag exactly at HEAD, when there is one (name as written, e.g. `v1.2.3`). + pub head_tag: Option, + /// Every tag of the repository, sorted lexically. + pub tags: Vec, + /// Last tag reachable from HEAD (used by [`GitOrigin::git_version`]). + pub last_tag: Option, + /// HEAD commit date as `%Y%m%d`. + pub head_date: Option, + /// HEAD commit short hash. + pub head_hash: Option, + /// Whether the worktree carries uncommitted changes (`git status + /// --porcelain` non-empty). + pub dirty: bool, +} + +impl GitOrigin { + /// Detect the git origin state of `dir`. Returns `None` when `dir` is + /// not inside a git work tree (or git cannot be run); every other + /// failure is fail-soft (the field stays empty). + pub fn detect(dir: &Path) -> Option { + // Inside a work tree? (fails outside git; prints "false" in a bare + // repository). + if git(dir, &["rev-parse", "--is-inside-work-tree"]).as_deref() != Some("true") { + return None; + } + + let mut origin = GitOrigin { + head_tag: git(dir, &["describe", "--tags", "--exact-match", "HEAD"]), + tags: git(dir, &["tag", "--list"]) + .map(|out| { + out.lines() + .map(str::to_string) + .filter(|line| !line.is_empty()) + .collect() + }) + .unwrap_or_default(), + last_tag: git(dir, &["describe", "--tags", "--abbrev=0"]), + head_date: git(dir, &["log", "-1", "--date=format:%Y%m%d", "--format=%cd"]), + head_hash: git(dir, &["rev-parse", "--short", "HEAD"]), + dirty: git(dir, &["status", "--porcelain"]).is_some_and(|out| !out.trim().is_empty()), + forge: None, + }; + + if let Some(url) = git(dir, &["remote", "get-url", "origin"]) + .or_else(|| git(dir, &["config", "--get", "remote.origin.url"])) + { + origin.forge = Forge::parse(&url); + } + + Some(origin) + } + + /// Upstream version named by the tag at HEAD: the tag with a leading + /// `v`/`V` (before a digit) stripped, and only when the result is a + /// plausible Debian upstream version (no `-`, which is the revision + /// separator). + pub fn head_tag_version(&self) -> Option { + sanitized_tag_version(self.head_tag.as_deref()?) + } + + /// Version suggestion for a HEAD between releases: + /// `+git.` (e.g. `1.2.3+git20260916.4b8a2f1`), + /// or `None` without a reachable tag, tag-strippable version, date or hash. + pub fn git_version(&self) -> Option { + let base = sanitized_tag_version(self.last_tag.as_deref()?)?; + let date = self.head_date.as_deref()?; + let hash = self.head_hash.as_deref()?; + Some(format!("{base}+git{date}.{hash}")) + } + + /// The repository tag naming `version` (a leading `v`/`V` on the tag is + /// ignored), whatever the state of HEAD. + pub fn tag_for_version(&self, version: &str) -> Option<&str> { + self.tags + .iter() + .map(String::as_str) + .find(|tag| sanitized_tag_version(tag).as_deref() == Some(version)) + } +} + +/// Strip a leading `v`/`V` (before a digit) off a tag name and keep only +/// results usable as a Debian upstream version: no `-` (which is the revision +/// separator) and no leading non-digit. +fn sanitized_tag_version(tag: &str) -> Option { + let stripped = tag + .strip_prefix(['v', 'V']) + .filter(|rest| rest.starts_with(|c: char| c.is_ascii_digit())) + .unwrap_or(tag); + if stripped.starts_with(|c: char| c.is_ascii_digit()) && !stripped.contains('-') { + Some(stripped.to_string()) + } else { + None + } +} + +/// Check out `tag` in the repository at `dir`, refusing a dirty worktree: +/// pkh never carries uncommitted changes across a checkout. Detached HEAD +/// is the expected outcome when packaging a release tag. +pub fn checkout_tag(dir: &Path, tag: &str) -> Result<(), Box> { + if let Some(status) = git(dir, &["status", "--porcelain"]) + && !status.trim().is_empty() + { + return Err(format!( + "The working tree of '{}' has uncommitted changes: commit or \ + stash them before checking out '{tag}' (pkh does not carry \ + changes across a checkout)", + dir.display() + ) + .into()); + } + let output = Command::new("git") + .args(["checkout", tag]) + .current_dir(dir) + .output() + .map_err(|e| format!("failed to run 'git checkout': {e}"))?; + if !output.status.success() { + return Err(format!( + "'git checkout {tag}' failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ) + .into()); + } + Ok(()) +} + +/// Run `git` with `args` in `dir`, returning its trimmed stdout when it +/// exits successfully (empty output stays an empty string). +fn git(dir: &Path, args: &[&str]) -> Option { + let output = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .ok()?; + output + .status + .success() + .then(|| String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + /// Whether the host has a usable git CLI (every fixture below needs it). + fn have_git() -> bool { + Command::new("git") + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()) + } + + /// Run git in `dir`, failing the test on error, with a deterministic + /// identity and no signing so host git config cannot break the fixture. + fn git(dir: &Path, args: &[&str]) { + let status = Command::new("git") + .args([ + "-c", + "user.name=Pkh Origin", + "-c", + "user.email=pkhorigin@example.invalid", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_DATE", "2026-09-15T12:00:00Z") + .env("GIT_COMMITTER_DATE", "2026-09-15T12:00:00Z") + .status() + .expect("git should be runnable"); + assert!(status.success(), "git {args:?} failed"); + } + + /// A repository with a single commit on 2026-09-15, returning its + /// short HEAD hash. + fn init_repo(dir: &Path) -> String { + git(dir, &["init", "-q"]); + std::fs::write(dir.join("file.txt"), "one\n").unwrap(); + git(dir, &["add", "file.txt"]); + git(dir, &["commit", "-q", "-m", "first"]); + git_out(dir, &["rev-parse", "--short", "HEAD"]) + } + + #[test] + fn forge_parses_remote_url_shapes() { + assert_eq!( + Forge::parse("https://github.com/foo/bar.git"), + Some(Forge::GitHub { + owner: "foo".into(), + repo: "bar".into() + }) + ); + assert_eq!( + Forge::parse("git@github.com:foo/bar.git"), + Some(Forge::GitHub { + owner: "foo".into(), + repo: "bar".into() + }) + ); + assert_eq!( + Forge::parse("git://github.com/foo/bar"), + Some(Forge::GitHub { + owner: "foo".into(), + repo: "bar".into() + }) + ); + assert_eq!( + Forge::parse("https://gitlab.com/foo/bar/-/tree/main"), + Some(Forge::GitLab { + owner: "foo".into(), + repo: "bar".into() + }) + ); + assert_eq!( + Forge::parse("ssh://git@gitlab.com/foo/bar.git"), + Some(Forge::GitLab { + owner: "foo".into(), + repo: "bar".into() + }) + ); + // Self-hosted GitLab instances and other hosts are not recognized. + assert_eq!(Forge::parse("https://gitlab.example.com/foo/bar.git"), None); + assert_eq!(Forge::parse("https://bitbucket.org/foo/bar.git"), None); + // Garbage. + assert_eq!(Forge::parse("not a url"), None); + assert_eq!(Forge::parse("https://github.com/onlyowner"), None); + } + + #[test] + fn forge_release_tarball_urls() { + let gh = Forge::GitHub { + owner: "foo".into(), + repo: "bar".into(), + }; + assert_eq!( + gh.release_tarball_urls("v1.2.3"), + vec![ + "https://codeload.github.com/foo/bar/tar.gz/refs/tags/v1.2.3".to_string(), + "https://github.com/foo/bar/archive/refs/tags/v1.2.3.tar.gz".to_string(), + ] + ); + let gl = Forge::GitLab { + owner: "foo".into(), + repo: "bar".into(), + }; + assert_eq!( + gl.release_tarball_urls("v1.2.3"), + vec!["https://gitlab.com/foo/bar/-/archive/v1.2.3/bar-v1.2.3.tar.gz".to_string()] + ); + } + + #[test] + fn sanitized_tag_versions() { + assert_eq!(sanitized_tag_version("v1.2.3"), Some("1.2.3".to_string())); + assert_eq!(sanitized_tag_version("V2.0"), Some("2.0".to_string())); + assert_eq!(sanitized_tag_version("1.2.3"), Some("1.2.3".to_string())); + // `v` followed by a non-digit is part of the name, not a marker. + assert_eq!(sanitized_tag_version("version-1"), None); + // `-` would collide with the Debian revision separator. + assert_eq!(sanitized_tag_version("v1.2.3-beta"), None); + } + + #[test] + fn detect_outside_a_repository_is_none() { + if !have_git() { + return; + } + let dir = tempdir().unwrap(); + assert_eq!(GitOrigin::detect(dir.path()), None); + // A bare repository is not a work tree either. + let bare = tempdir().unwrap(); + git(bare.path(), &["init", "-q", "--bare"]); + assert_eq!(GitOrigin::detect(bare.path()), None); + } + + #[test] + fn detect_plain_repo_without_tags_or_remote() { + if !have_git() { + return; + } + let dir = tempdir().unwrap(); + let hash = init_repo(dir.path()); + + let origin = GitOrigin::detect(dir.path()).expect("detected"); + assert_eq!(origin.forge, None); + assert_eq!(origin.head_tag, None); + assert!(origin.tags.is_empty()); + assert_eq!(origin.last_tag, None); + assert_eq!(origin.head_hash.as_deref(), Some(hash.as_str())); + assert_eq!(origin.head_date.as_deref(), Some("20260915")); + assert!(!origin.dirty); + // No tags: no version can be derived. + assert_eq!(origin.head_tag_version(), None); + assert_eq!(origin.git_version(), None); + } + + #[test] + fn detect_head_exactly_on_a_tag() { + if !have_git() { + return; + } + let dir = tempdir().unwrap(); + init_repo(dir.path()); + git(dir.path(), &["tag", "v1.2.3"]); + + let origin = GitOrigin::detect(dir.path()).expect("detected"); + assert_eq!(origin.head_tag.as_deref(), Some("v1.2.3")); + assert_eq!(origin.last_tag.as_deref(), Some("v1.2.3")); + assert_eq!(origin.head_tag_version().as_deref(), Some("1.2.3")); + // The query is the stripped version; the raw tag name is not one. + assert_eq!(origin.tag_for_version("1.2.3"), Some("v1.2.3")); + assert_eq!(origin.tag_for_version("v1.2.3"), None); + assert_eq!(origin.tag_for_version("9.9.9"), None); + } + + #[test] + fn detect_between_releases_derives_git_version() { + if !have_git() { + return; + } + let dir = tempdir().unwrap(); + init_repo(dir.path()); + git(dir.path(), &["tag", "v1.2.3"]); + std::fs::write(dir.path().join("file.txt"), "two\n").unwrap(); + git(dir.path(), &["add", "file.txt"]); + git(dir.path(), &["commit", "-q", "-m", "second"]); + let hash = git_out(dir.path(), &["rev-parse", "--short", "HEAD"]); + + let origin = GitOrigin::detect(dir.path()).expect("detected"); + assert_eq!(origin.head_tag, None); + assert_eq!(origin.last_tag.as_deref(), Some("v1.2.3")); + assert_eq!(origin.tag_for_version("1.2.3"), Some("v1.2.3")); + assert_eq!( + origin.git_version().as_deref(), + Some(format!("1.2.3+git20260915.{hash}").as_str()) + ); + } + + #[test] + fn detect_tracks_the_origin_remote() { + if !have_git() { + return; + } + let dir = tempdir().unwrap(); + init_repo(dir.path()); + // A local path remote: configuring it never touches the network. + git( + dir.path(), + &["remote", "add", "origin", "https://github.com/foo/bar.git"], + ); + + let origin = GitOrigin::detect(dir.path()).expect("detected"); + assert_eq!( + origin.forge, + Some(Forge::GitHub { + owner: "foo".into(), + repo: "bar".into() + }) + ); + } + + #[test] + fn detect_reports_a_dirty_worktree() { + if !have_git() { + return; + } + let dir = tempdir().unwrap(); + init_repo(dir.path()); + std::fs::write(dir.path().join("file.txt"), "uncommitted\n").unwrap(); + + let origin = GitOrigin::detect(dir.path()).expect("detected"); + assert!(origin.dirty); + } + + #[test] + fn git_version_skips_unusable_tags() { + // A tag carrying `-` cannot become an upstream version: the scheme + // degrades to None instead of proposing an invalid version. + let origin = GitOrigin { + last_tag: Some("1.2.3-beta".into()), + head_date: Some("20260915".into()), + head_hash: Some("abc1234".into()), + ..Default::default() + }; + assert_eq!(origin.git_version(), None); + // Date or hash missing: nothing to propose either. + assert_eq!( + GitOrigin { + last_tag: Some("v2.0".into()), + ..Default::default() + } + .git_version(), + None + ); + } + + /// `git` output trimmed (for the hash assertions above). + fn git_out(dir: &Path, args: &[&str]) -> String { + let output = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!(output.status.success(), "git {args:?} failed"); + String::from_utf8_lossy(&output.stdout).trim().to_string() + } + + /// The tag lookup ignores a `v` prefix in either direction. + #[test] + fn tag_for_version_matches_both_spellings() { + let origin = GitOrigin { + tags: vec!["1.0".to_string(), "v2.0".to_string()], + ..Default::default() + }; + assert_eq!(origin.tag_for_version("1.0"), Some("1.0")); + assert_eq!(origin.tag_for_version("2.0"), Some("v2.0")); + assert_eq!(origin.tag_for_version("3.0"), None); + } +} diff --git a/src/new/questions.rs b/src/new/questions.rs index 48c8add..6684618 100644 --- a/src/new/questions.rs +++ b/src/new/questions.rs @@ -23,7 +23,8 @@ use std::path::PathBuf; use indicatif::MultiProgress; use crate::new::detect::{self, Detection}; -use crate::new::options::{self, NewCli, NewOptions, SourceDir, TemplateId}; +use crate::new::options::{self, NewCli, NewOptions, SourceDir, SourceFormat, TemplateId}; +use crate::new::origin::GitOrigin; use crate::new::templates::{self, ProbeResult, ScaffoldOutcome}; use crate::ui::prompt; @@ -60,15 +61,17 @@ const SOURCE_LABEL: &str = "Where is the source code? "; const LICENSE_LABEL: &str = "License: "; const DIST_LABEL: &str = "Target distribution: "; const SERIES_LABEL: &str = "Target series: "; +const ORIG_LABEL: &str = "Where should the orig tarball come from? "; /// All select labels, so the separator test can check them in one place. #[cfg(test)] -const SELECT_LABELS: [&str; 5] = [ +const SELECT_LABELS: [&str; 6] = [ LANGUAGE_LABEL, SOURCE_LABEL, LICENSE_LABEL, DIST_LABEL, SERIES_LABEL, + ORIG_LABEL, ]; /// Run the `pkh new` flow: the wizard on an interactive terminal, plain @@ -88,8 +91,10 @@ fn is_interactive() -> bool { } /// The wizard question flow (spec "Proposed UX"), in order: -/// package name, language/build system, source location, upstream version, -/// Debian revision, one-line description, homepage, license, command name, +/// package name, language/build system, source location, upstream version +/// (with the checkout-tag offer when the version names an existing tag), +/// the orig-tarball origin (quilt + existing project only), Debian +/// revision, one-line description, homepage, license, command name, /// maintainer, target distribution, target series, metapackage Depends /// (`empty` template only), git init — then the summary screen and the /// final `Generate?` confirmation. Every question with an explicit flag @@ -205,17 +210,101 @@ async fn run_wizard(mut cli: NewCli) -> Result> { cli.source = Some(cwd.clone()); } - // 4. Upstream version. + // The source format this run will produce: explicit flags win, then the + // mode (skeleton → native, existing project → quilt). The wizard + // surfaces it in the summary; `options::resolve` re-derives it the same + // way for the non-interactive path. + let source_format = match (cli.native, cli.quilt) { + (true, true) => { + return Err("--native and --quilt are mutually exclusive".into()); + } + (true, false) => SourceFormat::Native, + (false, true) => SourceFormat::Quilt, + (false, false) => { + if cli.source.is_none() { + SourceFormat::Native + } else { + SourceFormat::Quilt + } + } + }; + + // Git origin of the packaged directory: drives the version default, the + // checkout-tag offer and the orig-tarball question. Purely local. + let packaged_dir = cli.source.clone(); + let mut origin = packaged_dir.as_deref().and_then(GitOrigin::detect); + + // 4. Upstream version: probed project version, then the tag HEAD sits + // on, then `+git.`, then 0.1.0. if cli.upstream_version.is_none() { let default = probe .as_ref() .and_then(|p| p.version.clone()) + .or_else(|| origin.as_ref().and_then(GitOrigin::head_tag_version)) + .or_else(|| origin.as_ref().and_then(GitOrigin::git_version)) .unwrap_or_else(|| "0.1.0".to_string()); let revision = cli.revision.unwrap_or(1); let answer = ask_text("Upstream version", &default, move |version: &str| { options::validate_upstream_version(version, revision) })?; - cli.upstream_version = Some(answer); + cli.upstream_version = Some(answer.clone()); + + // The typed version names an existing tag HEAD is not on: offer to + // check the release out (the packaging then matches the version). + if let Some(origin_state) = &origin + && let Some(tag) = origin_state.tag_for_version(&answer) + && Some(tag) != origin_state.head_tag.as_deref() + { + let question = format!( + "Version {answer} matches tag {tag}, but HEAD is not that \ + tag. Check out {tag} now?" + ); + if prompt::confirm(&question, false)? { + let dir = packaged_dir.as_deref().expect("origin implies a directory"); + crate::new::origin::checkout_tag(dir, tag)?; + log::info!( + "Checked out {tag} (detached HEAD — expected when \ + packaging a release)" + ); + // HEAD moved: the release/git-archive choices below must + // reflect the checkout. + origin = GitOrigin::detect(dir); + } + // Declined (or non-TTY): keep the current tree and the typed + // version. + } + } + + // 4b. Orig-tarball origin (quilt + existing project only): one select + // question, options pre-ordered by what the detection found. The + // release download is explicit network consent; a skeleton quilt + // tree has no upstream history and is only ever snapshotted. + if source_format == SourceFormat::Quilt && packaged_dir.is_some() { + let choices = orig_origin_choices(origin.as_ref()); + let labels: Vec = choices.iter().map(|(label, _)| label.clone()).collect(); + let head_tagged = origin + .as_ref() + .and_then(|o| o.head_tag.as_deref()) + .is_some(); + let default = if head_tagged { + labels[0].clone() + } else { + "Snapshot this working tree".to_string() + }; + let answer = select_from(ORIG_LABEL, &labels, &default, |answer| { + labels.contains(&answer.to_string()) + })?; + let chosen = choices + .iter() + .find(|(label, _)| label == &answer) + .map(|(_, value)| *value) + .expect("answer comes from the choice list"); + cli.orig_from = Some(chosen.to_string()); + if chosen == "path" { + let validator = |path: &str| options::validate_orig_path(path); + let path = prompt::text("Tarball path or URL", "", Some(&validator))?; + cli.orig_path = Some(path); + } } // 5. Debian revision. @@ -706,6 +795,37 @@ fn validate_revision_answer(answer: &str) -> Result<(), String> { } } +/// The choices of the orig-tarball-origin question for a quilt packaging of +/// an existing directory: `(menu label, --orig-from value)` pairs, +/// pre-ordered by what the git origin detection found — the release +/// download and `git archive` only exist when HEAD sits exactly on a tag, +/// the release download additionally needs a recognized forge. The working +/// tree snapshot is always available (and is the implicit default when HEAD +/// is not on a tag). +fn orig_origin_choices(origin: Option<&GitOrigin>) -> Vec<(String, &'static str)> { + let mut choices: Vec<(String, &'static str)> = Vec::new(); + if let Some(origin) = origin + && let Some(tag) = &origin.head_tag + { + if let Some(forge) = &origin.forge { + choices.push(( + format!( + "Download the upstream release tarball from {} ({tag})", + forge.host() + ), + "release", + )); + } + choices.push(( + format!("Create it from the git tag ({tag}, git archive)"), + "git", + )); + } + choices.push(("Use a tarball I provide".to_string(), "path")); + choices.push(("Snapshot this working tree".to_string(), "snapshot")); + choices +} + /// A `debian/watch` template for GitHub/GitLab-hosted projects; `None` when /// the homepage is not one of those hosts (the wizard skips the question). pub fn watch_template(homepage: Option<&str>) -> Option { @@ -784,6 +904,13 @@ pub fn summary_text(opts: &NewOptions, toolchain_pin: Option<&str>) -> String { lines.push(format!(" debian/rules {}", template.rules_dh_line())); } } + lines.push(format!( + " debian/source/format {}", + opts.source_format.deb_string() + )); + if let Some(orig) = &opts.orig { + lines.push(format!(" orig tarball {}", orig.label())); + } let distribution = if opts.release { opts.series.as_str() } else { @@ -850,7 +977,8 @@ mod tests { series: "resolute".into(), release: false, depends: Vec::new(), - native: false, + source_format: options::SourceFormat::Native, + orig: None, git: true, autopkgtest: false, pkg_config: false, @@ -945,6 +1073,92 @@ mod tests { assert!(watch_template(None).is_none()); } + /// The orig-origin choices are pre-ordered by what the detection found: + /// release download only with a forge, git archive only on a tag, + /// tarball/snapshot always; snapshot last (the off-tag default). + #[test] + fn orig_origin_choices_preorder_by_detection() { + use crate::new::origin::Forge; + let tagged_forge = GitOrigin { + forge: Some(Forge::GitHub { + owner: "foo".into(), + repo: "bar".into(), + }), + head_tag: Some("v1.4.0".into()), + ..Default::default() + }; + let choices = orig_origin_choices(Some(&tagged_forge)); + assert_eq!(choices.len(), 4); + assert_eq!(choices[0].1, "release"); + assert!(choices[0].0.contains("github.com")); + assert!(choices[0].0.contains("v1.4.0")); + assert_eq!(choices[1].1, "git"); + assert!(choices[1].0.contains("git archive")); + assert_eq!(choices[2].1, "path"); + assert_eq!(choices[3].1, "snapshot"); + + // Tag without a recognized forge: no download option. + let tagged = GitOrigin { + head_tag: Some("1.0.0".into()), + ..Default::default() + }; + let choices = orig_origin_choices(Some(&tagged)); + assert_eq!(choices.len(), 3); + assert_eq!(choices[0].1, "git"); + assert_eq!(choices[1].1, "path"); + assert_eq!(choices[2].1, "snapshot"); + + // Off a tag (or not even a repo): tarball + snapshot only. + let off_tag = GitOrigin { + last_tag: Some("v1.0.0".into()), + ..Default::default() + }; + let choices = orig_origin_choices(Some(&off_tag)); + assert_eq!(choices.len(), 2); + assert_eq!(choices[0].1, "path"); + assert_eq!(choices[1].1, "snapshot"); + assert_eq!(orig_origin_choices(None).len(), 2); + } + + /// The summary surfaces the derived source format and, for quilt, the + /// planned orig origin. + #[test] + fn summary_screen_shows_format_and_orig_origin() { + // Native skeleton: the format row, no orig row. + let text = summary_text(&opts(Tid::Shell), None); + assert!(text.contains("debian/source/format 3.0 (native)"), "{text}"); + assert!(!text.contains("orig tarball"), "{text}"); + + // Quilt over an existing project: both rows. + let mut quilt = opts(Tid::Shell); + quilt.source_dir = options::SourceDir::Here; + quilt.source_format = options::SourceFormat::Quilt; + quilt.orig = Some(options::OrigOrigin::GitArchive { + tag: "v1.4.0".to_string(), + }); + let text = summary_text(&quilt, None); + assert!(text.contains("debian/source/format 3.0 (quilt)"), "{text}"); + assert!( + text.contains("orig tarball git archive (v1.4.0)"), + "{text}" + ); + + // The release-download label of the origin matrix. + let mut release = quilt.clone(); + release.orig = Some(options::OrigOrigin::Release { + tag: "v0.14.0".to_string(), + forge: crate::new::origin::Forge::GitLab { + owner: "foo".into(), + repo: "bar".into(), + }, + }); + let text = summary_text(&release, None); + assert!( + text.contains("orig tarball release download (v0.14.0)"), + "{text}" + ); + } + #[test] fn summary_screen_skeleton() { let text = summary_text(&opts(Tid::Makefile), None); diff --git a/src/new/templates/autotools.rs b/src/new/templates/autotools.rs index b97e92b..4562dd8 100644 --- a/src/new/templates/autotools.rs +++ b/src/new/templates/autotools.rs @@ -129,7 +129,8 @@ mod tests { series: "resolute".into(), release: false, depends: Vec::new(), - native: false, + source_format: crate::new::options::SourceFormat::Quilt, + orig: Some(crate::new::options::OrigOrigin::Snapshot), git: true, autopkgtest: false, pkg_config: false, diff --git a/src/new/templates/cmake.rs b/src/new/templates/cmake.rs index 3acc091..fed4e8f 100644 --- a/src/new/templates/cmake.rs +++ b/src/new/templates/cmake.rs @@ -104,7 +104,8 @@ mod tests { series: "resolute".into(), release: false, depends: Vec::new(), - native: false, + source_format: crate::new::options::SourceFormat::Quilt, + orig: Some(crate::new::options::OrigOrigin::Snapshot), git: true, autopkgtest: false, pkg_config: false, diff --git a/src/new/templates/empty.rs b/src/new/templates/empty.rs index 8db53b5..944d20c 100644 --- a/src/new/templates/empty.rs +++ b/src/new/templates/empty.rs @@ -60,7 +60,8 @@ mod tests { series: "sid".into(), release: false, depends, - native: false, + source_format: crate::new::options::SourceFormat::Quilt, + orig: Some(crate::new::options::OrigOrigin::Snapshot), git: true, autopkgtest: false, pkg_config: false, diff --git a/src/new/templates/go.rs b/src/new/templates/go.rs index c9c4f38..310e9c9 100644 --- a/src/new/templates/go.rs +++ b/src/new/templates/go.rs @@ -134,7 +134,8 @@ mod tests { series: "resolute".into(), release: false, depends: Vec::new(), - native: false, + source_format: crate::new::options::SourceFormat::Quilt, + orig: Some(crate::new::options::OrigOrigin::Snapshot), git: true, autopkgtest: false, pkg_config: false, diff --git a/src/new/templates/makefile.rs b/src/new/templates/makefile.rs index 24e274d..ca5147c 100644 --- a/src/new/templates/makefile.rs +++ b/src/new/templates/makefile.rs @@ -139,7 +139,8 @@ mod tests { series: "resolute".into(), release: false, depends: Vec::new(), - native: false, + source_format: crate::new::options::SourceFormat::Quilt, + orig: Some(crate::new::options::OrigOrigin::Snapshot), git: true, autopkgtest: false, pkg_config: false, diff --git a/src/new/templates/meson.rs b/src/new/templates/meson.rs index 3cb9fd1..6ab43d0 100644 --- a/src/new/templates/meson.rs +++ b/src/new/templates/meson.rs @@ -118,7 +118,8 @@ mod tests { series: "resolute".into(), release: false, depends: Vec::new(), - native: false, + source_format: crate::new::options::SourceFormat::Quilt, + orig: Some(crate::new::options::OrigOrigin::Snapshot), git: true, autopkgtest: false, pkg_config: false, diff --git a/src/new/templates/mod.rs b/src/new/templates/mod.rs index 53b5fc8..b7cd2d5 100644 --- a/src/new/templates/mod.rs +++ b/src/new/templates/mod.rs @@ -54,12 +54,16 @@ impl OutputFile { /// What the template post-write hook did to the freshly written tree, /// threaded through [`super::scaffold`] so the flow can react (e.g. word /// the post-scaffold verification offer differently when vendoring failed). -#[derive(Debug, Clone, Copy, Default)] +#[derive(Debug, Clone, Default)] pub struct ScaffoldOutcome { /// The vendoring step did not complete (host `cargo` missing, `cargo /// vendor` failed, or the offline config could not be written): the /// package will not build until the dependencies are vendored manually. pub vendoring_failed: bool, + /// How the orig tarball was actually created (quilt only; `None` with + /// the native format, which has no orig tarball). Filled in by the + /// scaffold flow, not by the template hook. + pub orig_origin: Option, } /// Metadata extracted from an existing project by [`Template::probe`], used @@ -266,7 +270,8 @@ mod tests { series: "resolute".into(), release: false, depends: Vec::new(), - native: false, + source_format: crate::new::options::SourceFormat::Quilt, + orig: None, git: false, autopkgtest: false, pkg_config: false, @@ -335,7 +340,8 @@ mod tests { series: "resolute".into(), release: false, depends: Vec::new(), - native: false, + source_format: crate::new::options::SourceFormat::Quilt, + orig: None, git: false, autopkgtest: false, pkg_config: false, diff --git a/src/new/templates/python.rs b/src/new/templates/python.rs index e70a011..c878f7b 100644 --- a/src/new/templates/python.rs +++ b/src/new/templates/python.rs @@ -361,7 +361,8 @@ mod tests { series: "resolute".into(), release: false, depends: Vec::new(), - native: false, + source_format: crate::new::options::SourceFormat::Quilt, + orig: Some(crate::new::options::OrigOrigin::Snapshot), git: true, autopkgtest: false, pkg_config: false, diff --git a/src/new/templates/rust.rs b/src/new/templates/rust.rs index 54c2bfd..249996d 100644 --- a/src/new/templates/rust.rs +++ b/src/new/templates/rust.rs @@ -161,7 +161,10 @@ impl Template for Rust { // `--locked` now that the outcome is known, so the final rules use // it exactly when the lockfile exists. patch_rules_locked(tree)?; - Ok(ScaffoldOutcome { vendoring_failed }) + Ok(ScaffoldOutcome { + vendoring_failed, + orig_origin: None, + }) } } @@ -169,7 +172,10 @@ impl Template for Rust { /// offline source replacement. Returns whether the step completed; failures /// warn loudly and leave the tree for manual vendoring. I/O errors on the /// freshly written tree are the exception: they fail the scaffold. -fn vendor_dependencies(tree: &Path) -> Result> { +/// +/// `pub(crate)` because the `pkh build` re-vendor retry (see +/// [`crate::build`]'s `VendorDriftError` hook) reruns exactly this step. +pub(crate) fn vendor_dependencies(tree: &Path) -> Result> { let Some(cargo) = find_on_path("cargo") else { log::warn!( "cargo was not found on PATH: the Rust package will NOT build \ @@ -489,7 +495,8 @@ mod tests { series: "resolute".into(), release: false, depends: Vec::new(), - native: false, + source_format: crate::new::options::SourceFormat::Quilt, + orig: Some(crate::new::options::OrigOrigin::Snapshot), git: true, autopkgtest: false, pkg_config: false, diff --git a/src/new/templates/shell.rs b/src/new/templates/shell.rs index 8a6625b..03d9ec2 100644 --- a/src/new/templates/shell.rs +++ b/src/new/templates/shell.rs @@ -79,7 +79,8 @@ mod tests { series: "resolute".into(), release: false, depends: Vec::new(), - native: false, + source_format: crate::new::options::SourceFormat::Quilt, + orig: Some(crate::new::options::OrigOrigin::Snapshot), git: true, autopkgtest: false, pkg_config: false, diff --git a/src/new/verify.rs b/src/new/verify.rs index f2e3088..6dc2e40 100644 --- a/src/new/verify.rs +++ b/src/new/verify.rs @@ -71,9 +71,9 @@ pub fn verify(tree: &Path) -> Result<(), Box> { // Quilt packages need their orig tarball next to the tree. if format == super::debian::SOURCE_FORMAT_QUILT { - let uversion = parsed_version.upstream; + let uversion = super::orig::component_upstream_version(&parsed_version); let tarball = - super::debian::orig_tarball_path(tree, &source, &uversion).ok_or_else(|| { + super::debian::orig_tarball_path(tree, &source, uversion).ok_or_else(|| { format!( "cannot determine the parent directory of '{}'", tree.display() @@ -87,6 +87,29 @@ pub fn verify(tree: &Path) -> Result<(), Box> { ) .into()); } + + // Vendored rust: the orig-vendor component is required exactly when + // the main orig exists and the tree carries a generated, non-empty + // `vendor/` directory (native packages have no tarballs at all, and + // trees without vendoring must not demand the component either). + if super::orig::has_vendored_dir(tree) { + let component = super::orig::vendor_component_path(tree, &source, uversion) + .ok_or_else(|| { + format!( + "cannot determine the parent directory of '{}'", + tree.display() + ) + })?; + if !component.exists() { + return Err(format!( + "Quilt package with vendored dependencies but without \ + the orig-vendor component: '{}' is missing. Re-run \ + pkh new, or recreate it after re-vendoring.", + component.display() + ) + .into()); + } + } } Ok(()) @@ -95,7 +118,9 @@ pub fn verify(tree: &Path) -> Result<(), Box> { #[cfg(test)] mod tests { use super::*; - use crate::new::options::{License, NewOptions, SourceDir, TemplateId}; + use crate::new::options::{ + License, NewOptions, OrigOrigin, SourceDir, SourceFormat, TemplateId, + }; use tempfile::tempdir; fn opts() -> NewOptions { @@ -115,7 +140,8 @@ mod tests { series: "sid".into(), release: false, depends: Vec::new(), - native: false, + source_format: SourceFormat::Quilt, + orig: Some(OrigOrigin::Snapshot), git: false, autopkgtest: false, pkg_config: false, @@ -134,7 +160,7 @@ mod tests { files.extend(template.skeleton(opts)); files.extend(template.debian(opts)); crate::new::debian::write_files(&tree, &files).unwrap(); - if !opts.native { + if opts.source_format == SourceFormat::Quilt { crate::new::debian::create_orig_tarball(&tree, &opts.name, &opts.upstream_version) .unwrap(); } @@ -154,13 +180,54 @@ mod tests { let tree = scaffold_tree( dir.path(), &NewOptions { - native: true, + source_format: SourceFormat::Native, + orig: None, ..opts() }, ); verify(&tree).unwrap(); } + /// The orig-vendor component is demanded exactly when the tree carries + /// a non-empty vendor/ next to a quilt orig — and not otherwise. + #[test] + fn verify_demands_the_vendor_component_only_when_vendored() { + // Quilt tree with a populated vendor/: verify fails until the + // component exists. + let dir = tempdir().unwrap(); + let tree = scaffold_tree(dir.path(), &opts()); + std::fs::create_dir_all(tree.join("vendor/serde/src")).unwrap(); + std::fs::write(tree.join("vendor/serde/src/lib.rs"), "code").unwrap(); + let err = verify(&tree).unwrap_err().to_string(); + assert!(err.contains("orig-vendor"), "{err}"); + + // Creating the component fixes it. (The main orig was snapshotted + // before vendor/ existed, so it holds no vendor/ — dpkg-wise the + // component alone owns the vendored tree.) + crate::new::orig::create_vendor_component(&tree, "mytool", "0.1.0").unwrap(); + verify(&tree).unwrap(); + + // A quilt tree without vendor/ must not demand the component. + let dir = tempdir().unwrap(); + let tree = scaffold_tree(dir.path(), &opts()); + verify(&tree).unwrap(); + assert!(!dir.path().join("mytool_0.1.0.orig-vendor.tar.xz").exists()); + + // A native tree with vendor/ has no tarballs at all: no demand. + let dir = tempdir().unwrap(); + let tree = scaffold_tree( + dir.path(), + &NewOptions { + source_format: SourceFormat::Native, + orig: None, + ..opts() + }, + ); + std::fs::create_dir_all(tree.join("vendor/serde/src")).unwrap(); + std::fs::write(tree.join("vendor/serde/src/lib.rs"), "code").unwrap(); + verify(&tree).unwrap(); + } + #[test] fn verify_names_the_broken_file() { // Each case needs its own tempdir: scaffolding refuses to overwrite