new: add upstream-aware orig tarball origins and the orig-vendor component

src/new/mod.rs

@@ -3,8 +3,9 @@
 //!
 //! This module orchestrates a scaffold run: target directory checks, project
 //! detection, in-memory rendering of every file (all-or-nothing write), the
-//! template post-write hook (e.g. `cargo vendor`), orig tarball creation,
-//! git initialization, structural verification and the next-steps message.
+//! template post-write hook (e.g. `cargo vendor`), orig tarball creation
+//! (from the origin the run decided on — see [`origin`] and [`orig`]), git
+//! initialization, structural verification and the next-steps message.
 //! The interactive wizard ([`questions`]) fills a [`options::NewCli`] from
 //! its answers on a TTY and reuses [`options::resolve`] as the single source
 //! of truth for defaults and validation; without a TTY the same resolution
@@ -14,6 +15,8 @@ pub mod debian;
 pub mod detect;
 pub mod git;
 pub mod options;
+pub mod orig;
+pub mod origin;
 pub mod questions;
 pub mod templates;
 pub mod verify;
@@ -23,7 +26,7 @@ use std::time::Duration;
 
 use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 
-use options::NewOptions;
+use options::{NewOptions, SourceFormat};
 use templates::{OutputFile, ScaffoldOutcome};
 
 /// Scaffold a full Debian source tree from `opts`.
@@ -64,8 +67,8 @@ pub fn scaffold(
     pb.finish_and_clear();
     multi.remove(&pb);
 
-    if result.is_ok() {
-        print_success(&opts);
+    if let Ok(outcome) = &result {
+        print_success(&opts, outcome);
     }
     result
 }
@@ -127,7 +130,7 @@ fn scaffold_steps(opts: &NewOptions, pb: &ProgressBar) -> Result<ScaffoldOutcome
     }
 
     // Fail before writing anything when the orig tarball already exists.
-    if !opts.native
+    if opts.source_format == SourceFormat::Quilt
         && let Some(tarball) =
             debian::orig_tarball_path(&target, &opts.name, &opts.upstream_version_no_epoch())
         && tarball.exists()
@@ -182,15 +185,34 @@ fn scaffold_steps(opts: &NewOptions, pb: &ProgressBar) -> Result<ScaffoldOutcome
 
     // 5. Template post-write hook: run before the orig tarball is created,
     //    so files added here (rust: vendor/ + .cargo/config.toml) land
-    //    inside it. The outcome (e.g. a failed vendoring) is threaded back
-    //    to the caller.
+    //    inside it (or inside the orig-vendor component). The outcome
+    //    (e.g. a failed vendoring) is threaded back to the caller.
     pb.set_message("Running template hooks");
-    let outcome = template.post_write(opts, &target)?;
+    let mut outcome = template.post_write(opts, &target)?;
 
-    // 6. Orig tarball (quilt only).
-    if !opts.native {
+    // 6. Orig tarball (quilt only), from the origin the run decided on.
+    //    A vendored rust tree gets its `vendor/` directory moved into the
+    //    separate dpkg upstream component `orig-vendor`, regenerable
+    //    independently of the upstream sources (native packages have no
+    //    orig at all: vendor/ simply lives in the tree).
+    if opts.source_format == SourceFormat::Quilt {
         pb.set_message("Creating orig tarball");
-        debian::create_orig_tarball(&target, &opts.name, &opts.upstream_version_no_epoch())?;
+        let vendored_rust =
+            template.id() == options::TemplateId::Rust && orig::has_vendored_dir(&target);
+        let created = orig::create_orig(
+            &target,
+            &opts.name,
+            &opts.upstream_version_no_epoch(),
+            opts.orig
+                .as_ref()
+                .ok_or("internal error: a quilt scaffold needs an orig-tarball plan")?,
+            vendored_rust,
+        )?;
+        outcome.orig_origin = Some(created.label);
+        if vendored_rust {
+            pb.set_message("Creating the orig-vendor component");
+            orig::create_vendor_component(&target, &opts.name, &opts.upstream_version_no_epoch())?;
+        }
     }
 
     // 7. Git.
@@ -205,7 +227,7 @@ fn scaffold_steps(opts: &NewOptions, pb: &ProgressBar) -> Result<ScaffoldOutcome
 }
 
 /// The success message: what was created and the next steps.
-fn print_success(opts: &NewOptions) {
+fn print_success(opts: &NewOptions, outcome: &ScaffoldOutcome) {
     let target =
         opts.target_dir(&std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from(".")));
     // `display_path` yields an empty string when the target is the cwd
@@ -226,6 +248,9 @@ fn print_success(opts: &NewOptions) {
         opts.series,
         opts.template
     );
+    if let Some(orig_origin) = &outcome.orig_origin {
+        log::info!("Orig tarball: {orig_origin}");
+    }
     log::info!("Next steps:");
     log::info!("  cd {}", if display.is_empty() { "." } else { &display });
     if opts.release {
@@ -244,11 +269,19 @@ fn print_success(opts: &NewOptions) {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::new::options::{License, SourceDir, TemplateId};
+    use crate::new::options::{License, OrigOrigin, SourceDir, TemplateId};
     use serial_test::serial;
     use tempfile::tempdir;
 
     fn opts(template: TemplateId, name: &str, source_dir: SourceDir) -> NewOptions {
+        // Mirror the resolve() derivation: skeletons are native by default,
+        // existing projects quilt with a working-tree snapshot orig.
+        let (source_format, orig) = match source_dir {
+            SourceDir::Skeleton => (SourceFormat::Native, None),
+            SourceDir::Here | SourceDir::Path(_) => {
+                (SourceFormat::Quilt, Some(OrigOrigin::Snapshot))
+            }
+        };
         NewOptions {
             name: name.to_string(),
             template,
@@ -265,7 +298,8 @@ mod tests {
             series: "resolute".into(),
             release: false,
             depends: Vec::new(),
-            native: false,
+            source_format,
+            orig,
             git: false,
             autopkgtest: false,
             pkg_config: false,
@@ -305,7 +339,6 @@ mod tests {
             "debian/rules",
             "debian/copyright",
             "debian/source/format",
-            "debian/source/local-options",
             "debian/.gitignore",
             "debian/install",
             "mytool.sh",
@@ -340,15 +373,14 @@ mod tests {
         assert_eq!(version, "0.1.0-1");
         assert_eq!(distribution, "UNRELEASED");
 
-        // source/format + local-options.
+        // A fresh skeleton is 3.0 (native) by default: no local-options and
+        // no orig tarball anywhere.
         assert_eq!(
             std::fs::read_to_string(tree.join("debian/source/format")).unwrap(),
-            "3.0 (quilt)\n"
-        );
-        assert_eq!(
-            std::fs::read_to_string(tree.join("debian/source/local-options")).unwrap(),
-            "single-debian-patch\n"
+            "3.0 (native)\n"
         );
+        assert!(!tree.join("debian/source/local-options").exists());
+        assert!(!dir.path().join("mytool_0.1.0.orig.tar.xz").exists());
 
         // install mapping.
         assert_eq!(
@@ -360,6 +392,28 @@ mod tests {
         let gitignore = std::fs::read_to_string(tree.join(".gitignore")).unwrap();
         assert!(gitignore.contains("*.deb"));
         assert!(gitignore.contains("target/"));
+    }
+
+    /// A skeleton forced to quilt keeps the snapshot behavior: orig tarball
+    /// with the skeleton files, `debian/` excluded, local-options present.
+    #[test]
+    #[serial]
+    fn scaffold_skeleton_forced_quilt_snapshots_the_tree() {
+        let dir = tempdir().unwrap();
+        let mut o = opts(TemplateId::Shell, "mytool", SourceDir::Skeleton);
+        o.source_format = SourceFormat::Quilt;
+        o.orig = Some(OrigOrigin::Snapshot);
+        scaffold_in(dir.path(), o).unwrap();
+
+        let tree = dir.path().join("mytool");
+        assert_eq!(
+            std::fs::read_to_string(tree.join("debian/source/format")).unwrap(),
+            "3.0 (quilt)\n"
+        );
+        assert_eq!(
+            std::fs::read_to_string(tree.join("debian/source/local-options")).unwrap(),
+            "single-debian-patch\n"
+        );
 
         // Orig tarball: contains the skeleton file, excludes debian/.
         let tarball = dir.path().join("mytool_0.1.0.orig.tar.xz");
@@ -403,29 +457,11 @@ mod tests {
         );
         assert_eq!(control.binaries[0].get("Architecture"), Some("all"));
         // No install file, no build-system skeleton: the README stub only.
+        // A native skeleton carries no orig tarball: the README simply
+        // lives in the tree.
         assert!(!tree.join("debian/install").exists());
         assert!(tree.join("README").exists());
-        // The tarball excludes debian/ but carries the README.
-        let tarball = dir.path().join("metapkg_0.1.0.orig.tar.xz");
-        let mut archive = tar::Archive::new(xz2::read::XzDecoder::new(
-            std::fs::File::open(&tarball).unwrap(),
-        ));
-        let names: Vec<String> = archive
-            .entries()
-            .unwrap()
-            .map(|entry| {
-                entry
-                    .unwrap()
-                    .path()
-                    .unwrap()
-                    .to_string_lossy()
-                    .into_owned()
-            })
-            .collect();
-        assert!(
-            names.iter().any(|n| n == "metapkg-0.1.0/README"),
-            "{names:?}"
-        );
+        assert!(!dir.path().join("metapkg_0.1.0.orig.tar.xz").exists());
 
         // Empty base flavor: no depends, no Depends field.
         let dir = tempdir().unwrap();
@@ -527,14 +563,13 @@ mod tests {
         .unwrap_err();
         assert!(err.to_string().contains("not empty"), "{err}");
 
-        // Existing orig tarball: nothing gets written.
+        // Existing orig tarball (quilt only): nothing gets written.
         let dir = tempdir().unwrap();
         std::fs::write(dir.path().join("mytool_0.1.0.orig.tar.xz"), b"old").unwrap();
-        let err = scaffold_in(
-            dir.path(),
-            opts(TemplateId::Shell, "mytool", SourceDir::Skeleton),
-        )
-        .unwrap_err();
+        let mut o = opts(TemplateId::Shell, "mytool", SourceDir::Skeleton);
+        o.source_format = SourceFormat::Quilt;
+        o.orig = Some(OrigOrigin::Snapshot);
+        let err = scaffold_in(dir.path(), o).unwrap_err();
         assert!(err.to_string().contains("already exists"), "{err}");
         assert!(!dir.path().join("mytool/debian/control").exists());
 
@@ -552,13 +587,17 @@ mod tests {
         assert!(err.to_string().contains("does not exist"), "{err}");
     }
 
+    /// The skeleton default is `3.0 (native)`: no orig tarball, no
+    /// `debian/source/local-options`.
     #[test]
     #[serial]
-    fn scaffold_native_skips_tarball_and_local_options() {
+    fn scaffold_skeleton_defaults_to_native() {
         let dir = tempdir().unwrap();
-        let mut o = opts(TemplateId::Shell, "nativepkg", SourceDir::Skeleton);
-        o.native = true;
-        scaffold_in(dir.path(), o).unwrap();
+        scaffold_in(
+            dir.path(),
+            opts(TemplateId::Shell, "nativepkg", SourceDir::Skeleton),
+        )
+        .unwrap();
 
         let tree = dir.path().join("nativepkg");
         assert!(!dir.path().join("nativepkg_0.1.0.orig.tar.xz").exists());
@@ -569,17 +608,17 @@ mod tests {
         );
     }
 
-    /// End-to-end rust skeleton: the vendoring hook runs before the orig
-    /// tarball is created, so `.cargo/` (and `vendor/` when dependencies
-    /// exist) travel inside it. The vendoring step needs host cargo; on a
-    /// cargo-less host the scaffold still succeeds with a warning and a
-    /// `vendoring_failed` outcome. Keyed against the `RUSTUP_TOOLCHAIN`
-    /// tests of the rust template: they mutate the process-global
-    /// environment the cargo shim would pick up mid-vendoring.
+    /// End-to-end rust skeleton: the vendoring hook runs over the tree
+    /// (native format: everything simply lives in the tree, no orig
+    /// tarball). The vendoring step needs host cargo; on a cargo-less host
+    /// the scaffold still succeeds with a warning and a `vendoring_failed`
+    /// outcome. Keyed against the `RUSTUP_TOOLCHAIN` tests of the rust
+    /// template: they mutate the process-global environment the cargo shim
+    /// would pick up mid-vendoring.
     #[test]
     #[serial]
     #[serial(RUSTUP_TOOLCHAIN)]
-    fn scaffold_rust_skeleton_vendors_before_tarball() {
+    fn scaffold_rust_skeleton_vendors_into_the_tree() {
         let dir = tempdir().unwrap();
         let outcome = scaffold_in(
             dir.path(),
@@ -589,6 +628,9 @@ mod tests {
 
         let has_cargo = crate::new::templates::find_on_path("cargo").is_some();
         assert_eq!(outcome.vendoring_failed, !has_cargo);
+        // Native skeleton: no orig tarball at all.
+        assert_eq!(outcome.orig_origin, None);
+        assert!(!dir.path().join("mytool_0.1.0.orig.tar.xz").exists());
 
         let tree = dir.path().join("mytool");
         assert!(tree.join("Cargo.toml").exists());
@@ -613,14 +655,60 @@ mod tests {
             Some("debhelper-compat (= 13),\ncargo:native,\nrustc:native")
         );
 
-        // The offline config exists when host cargo vendored the skeleton,
-        // and both it and the skeleton land inside the orig tarball.
+        // The offline config exists in the tree when host cargo vendored
+        // the skeleton.
         if has_cargo {
             let config = std::fs::read_to_string(tree.join(".cargo/config.toml")).unwrap();
             assert!(config.contains("[source.crates-io]"), "{config}");
             assert!(config.contains("[net]\noffline = true"), "{config}");
         }
+    }
 
+    /// End-to-end vendored rust quilt package: the vendoring hook creates
+    /// `vendor/`, the main orig excludes it, the dpkg upstream component
+    /// `orig-vendor` carries it, and the real source build
+    /// (`run_source_build`) lists BOTH tarballs in the `.dsc` and succeeds.
+    /// Needs host cargo with a working crates.io sync (skipped gracefully
+    /// when either is unavailable) and the local dpkg tools.
+    #[test]
+    #[serial]
+    #[serial(RUSTUP_TOOLCHAIN)]
+    fn scaffold_rust_quilt_with_deps_vendors_into_a_component() {
+        let dir = tempdir().unwrap();
+        let source = dir.path().join("mytool");
+        std::fs::create_dir_all(source.join("src")).unwrap();
+        // `libc` resolves from the host's cargo registry cache; vendoring
+        // it needs one crates.io index sync.
+        std::fs::write(
+            source.join("Cargo.toml"),
+            "[package]\n\
+             name = \"mytool\"\n\
+             version = \"0.1.0\"\n\
+             edition = \"2021\"\n\
+             \n\
+             [dependencies]\n\
+             libc = \"0.2\"\n",
+        )
+        .unwrap();
+        std::fs::write(source.join("src/main.rs"), "fn main() {}\n").unwrap();
+
+        let mut o = opts(TemplateId::Rust, "mytool", SourceDir::Path(source.clone()));
+        o.source_format = SourceFormat::Quilt;
+        o.orig = Some(OrigOrigin::Snapshot);
+        let outcome = scaffold_in(dir.path(), o).unwrap();
+
+        if crate::new::templates::find_on_path("cargo").is_none() || outcome.vendoring_failed {
+            // No cargo on this host or the crates.io sync failed: the
+            // vendoring guarantees of this test cannot hold.
+            log::warn!("cargo/crates.io unavailable; skipping the vendored component checks");
+            return;
+        }
+        assert_eq!(
+            outcome.orig_origin,
+            Some("working tree snapshot".to_string())
+        );
+
+        // The main orig excludes vendor/ but carries the upstream files.
         let tarball = dir.path().join("mytool_0.1.0.orig.tar.xz");
         let mut archive = tar::Archive::new(xz2::read::XzDecoder::new(
             std::fs::File::open(&tarball).unwrap(),
@@ -641,21 +729,58 @@ mod tests {
             names.iter().any(|n| n == "mytool-0.1.0/Cargo.toml"),
             "{names:?}"
         );
+        assert!(!names.iter().any(|n| n.contains("vendor")), "{names:?}");
+
+        // The component carries vendor/ under a top-level vendor/ dir.
+        let component = dir.path().join("mytool_0.1.0.orig-vendor.tar.xz");
+        assert!(component.exists());
+        let mut archive = tar::Archive::new(xz2::read::XzDecoder::new(
+            std::fs::File::open(&component).unwrap(),
+        ));
+        let names: Vec<String> = archive
+            .entries()
+            .unwrap()
+            .map(|entry| {
+                entry
+                    .unwrap()
+                    .path()
+                    .unwrap()
+                    .to_string_lossy()
+                    .into_owned()
+            })
+            .collect();
         assert!(
-            names.iter().any(|n| n == "mytool-0.1.0/src/main.rs"),
+            names.iter().any(|n| n.starts_with("vendor/libc/")),
             "{names:?}"
         );
-        if has_cargo {
-            assert!(
-                names.iter().any(|n| n == "mytool-0.1.0/.cargo/config.toml"),
-                "{names:?}"
-            );
-        }
-        assert!(!names.iter().any(|n| n.contains("debian")), "{names:?}");
+
+        // The real source build: the .dsc references both tarballs.
+        let output = crate::build::run_source_build(
+            &source,
+            &crate::build::SourceBuildOptions::default(),
+            None,
+        )
+        .unwrap();
+        let dsc = std::fs::read_to_string(&output.dsc).unwrap();
+        assert!(dsc.contains("mytool_0.1.0.orig.tar.xz"), "{dsc}");
+        assert!(dsc.contains("mytool_0.1.0.orig-vendor.tar.xz"), "{dsc}");
+        assert!(
+            output
+                .tarballs
+                .iter()
+                .any(|t| t.ends_with("mytool_0.1.0.orig.tar.xz"))
+        );
+        assert!(
+            output
+                .tarballs
+                .iter()
+                .any(|t| t.ends_with("mytool_0.1.0.orig-vendor.tar.xz"))
+        );
+        assert!(!output.signed);
     }
 
-    /// End-to-end python skeleton: pyproject-based Build-Depends and the
-    /// module skeleton inside the orig tarball.
+    /// End-to-end python skeleton: pyproject-based Build-Depends (native
+    /// skeleton: the upstream files live in the tree, no orig tarball).
     #[test]
     #[serial]
     fn scaffold_python_skeleton_tree() {
@@ -679,44 +804,22 @@ mod tests {
         let rules = std::fs::read_to_string(tree.join("debian/rules")).unwrap();
         assert!(rules.contains("%:\n\tdh $@ --with python3 --buildsystem=pybuild\n"));
 
-        let tarball = dir.path().join("mytool_0.1.0.orig.tar.xz");
-        let mut archive = tar::Archive::new(xz2::read::XzDecoder::new(
-            std::fs::File::open(&tarball).unwrap(),
-        ));
-        let names: Vec<String> = archive
-            .entries()
-            .unwrap()
-            .map(|entry| {
-                entry
-                    .unwrap()
-                    .path()
-                    .unwrap()
-                    .to_string_lossy()
-                    .into_owned()
-            })
-            .collect();
-        assert!(
-            names.iter().any(|n| n == "mytool-0.1.0/pyproject.toml"),
-            "{names:?}"
-        );
-        assert!(
-            names.iter().any(|n| n == "mytool-0.1.0/mytool/__init__.py"),
-            "{names:?}"
-        );
+        assert!(tree.join("pyproject.toml").exists());
+        assert!(tree.join("mytool/__init__.py").exists());
+        assert!(!dir.path().join("mytool_0.1.0.orig.tar.xz").exists());
     }
 
-    /// End-to-end: the scaffolded shell tree passes the real source build
-    /// (`dpkg-source` and friends, same prerequisites as the differential
-    /// tests).
+    /// End-to-end: a quilt tree (Here mode over an existing source) passes
+    /// the real source build (`dpkg-source` and friends, same prerequisites
+    /// as the differential tests).
     #[test]
     #[serial]
     fn scaffold_then_source_build_produces_artifacts() {
         let dir = tempdir().unwrap();
-        scaffold_in(
-            dir.path(),
-            opts(TemplateId::Shell, "mytool", SourceDir::Skeleton),
-        )
-        .unwrap();
+        let tree = dir.path().join("mytool");
+        std::fs::create_dir_all(&tree).unwrap();
+        std::fs::write(tree.join("run.sh"), "#!/bin/sh\necho hi\n").unwrap();
+        scaffold_in(&tree, opts(TemplateId::Shell, "mytool", SourceDir::Here)).unwrap();
 
         let output = crate::build::run_source_build(
             &dir.path().join("mytool"),
@@ -736,4 +839,36 @@ mod tests {
         // UNRELEASED: nothing is signed.
         assert!(!output.signed);
     }
+
+    /// End-to-end native: a self-authored skeleton (3.0 (native), no orig
+    /// tarball) builds into a .dsc without any tarball at all.
+    #[test]
+    #[serial]
+    fn scaffold_native_then_source_build_needs_no_tarball() {
+        let dir = tempdir().unwrap();
+        scaffold_in(
+            dir.path(),
+            opts(TemplateId::Shell, "mytool", SourceDir::Skeleton),
+        )
+        .unwrap();
+
+        let output = crate::build::run_source_build(
+            &dir.path().join("mytool"),
+            &crate::build::SourceBuildOptions::default(),
+            None,
+        )
+        .unwrap();
+        assert!(output.dsc.exists(), "{:?} missing", output.dsc);
+        assert!(output.changes.exists(), "{:?} missing", output.changes);
+        // 3.0 (native): one self-contained source tarball (debian/ inside),
+        // but no ORIG tarball.
+        assert_eq!(output.tarballs.len(), 1, "{:?}", output.tarballs);
+        assert!(
+            !output.tarballs[0]
+                .file_name()
+                .is_some_and(|name| name.to_string_lossy().contains("orig")),
+            "{:?}",
+            output.tarballs
+        );
+    }
 }