new: detect downloaded tarballs by magic bytes, not extension

src/new/orig.rs

@@ -363,10 +363,89 @@ fn download_to_temp(url: &str) -> Result<PathBuf, Box<dyn std::error::Error>> {
     Ok(temp)
 }
 
+/// Container/compression format of a tarball, detected from its content
+/// or, as a fallback, from its name.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum TarballFormat {
+    /// gzip stream (`1f 8b`), decoded natively.
+    Gzip,
+    /// xz stream (`fd 37 7a 58 5a 00`), decoded natively.
+    Xz,
+    /// bzip2 stream (`42 5a 68`, "BZh"), decompressed through the host
+    /// `bzip2` binary.
+    Bzip2,
+    /// Uncompressed tar (`ustar` at offset 257).
+    PlainTar,
+}
+
+/// Sniff the format of the tarball at `path` from its leading magic bytes
+/// rather than its name: gzip (`1f 8b`), xz (`fd 37 7a 58 5a 00`), bzip2
+/// (`"BZh"`) and plain tar (`ustar` at offset 257) are recognized.
+/// Returns `None` when no signature matches (or the file cannot be read).
+fn sniff_tarball_format(path: &Path) -> Option<TarballFormat> {
+    // 257 + len("ustar"): the longest signature lives at that offset.
+    let mut head = [0u8; 262];
+    let mut file = std::fs::File::open(path).ok()?;
+    let mut filled = 0;
+    while filled < head.len() {
+        match file.read(&mut head[filled..]) {
+            Ok(0) => break,
+            Ok(n) => filled += n,
+            Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
+            Err(_) => return None,
+        }
+    }
+    if head.starts_with(&[0x1f, 0x8b]) {
+        Some(TarballFormat::Gzip)
+    } else if head.starts_with(&[0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00]) {
+        Some(TarballFormat::Xz)
+    } else if head.starts_with(b"BZh") {
+        Some(TarballFormat::Bzip2)
+    } else if filled == head.len() && head[257..].starts_with(b"ustar") {
+        Some(TarballFormat::PlainTar)
+    } else {
+        None
+    }
+}
+
+/// Map a tarball file NAME to its format — the fallback for local
+/// user-provided files whose content carries no recognizable magic
+/// signature (downloads are always sniffed first and never rejected for a
+/// missing extension). Recognizes `.tar`, `.tar.gz`/`.tgz`,
+/// `.tar.xz`/`.txz` and `.tar.bz2`/`.tbz2` (case-insensitively).
+fn format_from_extension(path: &Path) -> Option<TarballFormat> {
+    let full = path
+        .file_name()
+        .and_then(|name| name.to_str())
+        .unwrap_or_default()
+        .to_ascii_lowercase();
+    let extension = path
+        .extension()
+        .and_then(|ext| ext.to_str())
+        .unwrap_or_default()
+        .to_ascii_lowercase();
+    if full.ends_with(".tar.gz") || full.ends_with(".tgz") {
+        Some(TarballFormat::Gzip)
+    } else if full.ends_with(".tar.xz") || full.ends_with(".txz") {
+        Some(TarballFormat::Xz)
+    } else if full.ends_with(".tar.bz2") || full.ends_with(".tbz2") {
+        Some(TarballFormat::Bzip2)
+    } else if extension == "tar" {
+        Some(TarballFormat::PlainTar)
+    } else {
+        None
+    }
+}
+
 /// Open `path` as a (possibly compressed) tar stream and repack it under
 /// the `<name>-<uver>/` prefix into `../<name>_<uver>.orig.tar.xz`.
-/// `.tar.bz2`/`.tbz2` inputs are decompressed through the host `bzip2`
+/// The decompression route is decided by the file's CONTENT (magic bytes),
-/// binary (pkh carries no bzip2 codec); gz and xz are decoded natively.
+/// not its name: downloaded streams land in an extensionless temporary
+/// file and must not be rejected for a missing extension. Only local
+/// user-provided files whose content carries no recognizable signature
+/// fall back to the file-name extension. bzip2 content is decompressed
+/// through the host `bzip2` binary (pkh carries no bzip2 codec); gz and
+/// xz are decoded natively.
 fn repack_tarball_file(
     path: &Path,
     name: &str,
@@ -387,49 +466,44 @@ fn repack_tarball_file(
         .into());
     }
 
-    let extension = path
+    // Content decides the decompression route; the name is only a fallback
-        .extension()
+    // for local files with no recognizable magic (an unrecognized download
-        .and_then(|ext| ext.to_str())
+    // is rejected on its content, never on a missing extension).
-        .unwrap_or_default()
+    let format = sniff_tarball_format(path)
-        .to_ascii_lowercase();
+        .or_else(|| format_from_extension(path))
-    let full = path
+        .ok_or_else(|| {
-        .file_name()
+            format!(
-        .and_then(|name| name.to_str())
+                "'{}' does not look like a tarball: unrecognized content \
-        .unwrap_or_default()
+                 (expected gzip, xz, bzip2 or plain tar)",
-        .to_ascii_lowercase();
+                path.display()
+            )
+        })?;
+    log::debug!("Tarball '{}' detected as {format:?}", path.display());
 
     let mut bzip2_child: Option<std::process::Child> = None;
-    let reader: Box<dyn Read> = if full.ends_with(".tar.gz") || full.ends_with(".tgz") {
+    let reader: Box<dyn Read> = match format {
-        Box::new(flate2::read::GzDecoder::new(std::fs::File::open(path)?))
+        TarballFormat::Gzip => Box::new(flate2::read::GzDecoder::new(std::fs::File::open(path)?)),
-    } else if full.ends_with(".tar.xz") || full.ends_with(".txz") {
+        TarballFormat::Xz => Box::new(xz2::read::XzDecoder::new(std::fs::File::open(path)?)),
-        Box::new(xz2::read::XzDecoder::new(std::fs::File::open(path)?))
+        TarballFormat::Bzip2 => {
-    } else if full.ends_with(".tar.bz2") || full.ends_with(".tbz2") {
+            let mut child = Command::new("bzip2")
-        let mut child = Command::new("bzip2")
+                .arg("-dc")
-            .arg("-dc")
+                .arg(path)
-            .arg(path)
+                .stdout(Stdio::piped())
-            .stdout(Stdio::piped())
+                .spawn()
-            .spawn()
+                .map_err(|e| {
-            .map_err(|e| {
+                    format!(
-                format!(
+                        "'.tar.bz2' tarballs need the bzip2 binary on PATH to be \
-                    "'.tar.bz2' tarballs need the bzip2 binary on PATH to be \
+                         repacked: {e}"
-                     repacked: {e}"
+                    )
-                )
+                })?;
-            })?;
+            let stdout = child
-        let stdout = child
+                .stdout
-            .stdout
+                .take()
-            .take()
+                .ok_or_else(|| "bzip2 produced no output".to_string())?;
-            .ok_or_else(|| "bzip2 produced no output".to_string())?;
+            bzip2_child = Some(child);
-        bzip2_child = Some(child);
+            Box::new(stdout)
-        Box::new(stdout)
+        }
-    } else if extension == "tar" {
+        TarballFormat::PlainTar => Box::new(std::fs::File::open(path)?),
-        Box::new(std::fs::File::open(path)?)
-    } else {
-        return Err(format!(
-            "'{}' does not look like a tarball: expected .tar, .tar.gz, \
-             .tgz, .tar.bz2, .tbz2 or .tar.xz",
-            path.display()
-        )
-        .into());
     };
 
     let result = repack_tar_stream(reader, name, upstream_version, &dest);
@@ -543,12 +617,12 @@ mod tests {
             .collect()
     }
 
-    /// Build a gz tarball with the given entries (path → contents), under
+    /// Append the fixture entries (plus the `oldpkg-1.0/` top-level
-    /// the `oldpkg-1.0/` top-level directory.
+    /// directory) to any tar builder.
-    fn write_gz_fixture(path: &Path, entries: &[(&str, &str)]) {
+    fn append_fixture_entries<W: std::io::Write>(
-        let file = std::fs::File::create(path).unwrap();
+        builder: &mut tar::Builder<W>,
-        let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
+        entries: &[(&str, &str)],
-        let mut builder = tar::Builder::new(encoder);
+    ) {
         let mut header = tar::Header::new_gnu();
         header.set_entry_type(tar::EntryType::Directory);
         header.set_size(0);
@@ -570,9 +644,62 @@ mod tests {
                 )
                 .unwrap();
         }
+    }
+
+    /// Build a gz tarball with the given entries (path → contents), under
+    /// the `oldpkg-1.0/` top-level directory.
+    fn write_gz_fixture(path: &Path, entries: &[(&str, &str)]) {
+        let file = std::fs::File::create(path).unwrap();
+        let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
+        let mut builder = tar::Builder::new(encoder);
+        append_fixture_entries(&mut builder, entries);
         builder.into_inner().unwrap();
     }
 
+    /// Build an uncompressed tar with the given entries, under the
+    /// `oldpkg-1.0/` top-level directory.
+    fn write_plain_tar_fixture(path: &Path, entries: &[(&str, &str)]) {
+        let mut builder = tar::Builder::new(std::fs::File::create(path).unwrap());
+        append_fixture_entries(&mut builder, entries);
+        builder.finish().unwrap();
+    }
+
+    /// Build an xz tarball with the given entries, under the `oldpkg-1.0/`
+    /// top-level directory.
+    fn write_xz_fixture(path: &Path, entries: &[(&str, &str)]) {
+        let file = std::fs::File::create(path).unwrap();
+        let encoder = xz2::write::XzEncoder::new(file, 1);
+        let mut builder = tar::Builder::new(encoder);
+        append_fixture_entries(&mut builder, entries);
+        builder.finish().unwrap();
+    }
+
+    /// Build a bzip2 tarball with the given entries by piping a plain tar
+    /// through the host `bzip2` binary (pkh carries no bzip2 codec).
+    /// Returns false when the binary is unavailable; callers skip the
+    /// bzip2-specific assertions then.
+    fn write_bzip2_fixture(path: &Path, entries: &[(&str, &str)]) -> bool {
+        if std::process::Command::new("bzip2")
+            .arg("--version")
+            .output()
+            .is_err()
+        {
+            return false;
+        }
+        let plain_path = path.with_extension("tar");
+        write_plain_tar_fixture(&plain_path, entries);
+        let compressed = std::fs::File::create(path).unwrap();
+        let ok = std::process::Command::new("bzip2")
+            .arg("-zc")
+            .arg(&plain_path)
+            .stdout(Stdio::from(compressed))
+            .status()
+            .map(|status| status.success())
+            .unwrap_or(false);
+        let _ = std::fs::remove_file(&plain_path);
+        ok
+    }
+
     #[test]
     fn repack_rewrites_the_top_level_prefix() {
         let dir = tempfile::tempdir().unwrap();
@@ -684,6 +811,177 @@ mod tests {
         assert!(tarball_names(&dest).contains(&"mytool-2.1/g.txt".to_string()));
     }
 
+    /// Every tarball format is recognized from its real content alone:
+    /// gzip (`1f 8b`), xz (`fd 37 7a 58 5a 00`), bzip2 (`"BZh"`) and
+    /// plain tar (`ustar` at offset 257). Garbage and truncated files
+    /// match nothing.
+    #[test]
+    fn sniff_detects_every_magic_signature() {
+        let dir = tempfile::tempdir().unwrap();
+
+        let gz = dir.path().join("f.tar.gz");
+        write_gz_fixture(&gz, &[("a.txt", "A")]);
+        assert_eq!(sniff_tarball_format(&gz), Some(TarballFormat::Gzip));
+
+        let xz = dir.path().join("f.tar.xz");
+        write_xz_fixture(&xz, &[("a.txt", "A")]);
+        assert_eq!(sniff_tarball_format(&xz), Some(TarballFormat::Xz));
+
+        // Real bzip2 bytes when the host binary exists, else the raw
+        // "BZh" header the sniffer keys on.
+        let bz2 = dir.path().join("f.tar.bz2");
+        if write_bzip2_fixture(&bz2, &[("a.txt", "A")]) {
+            assert_eq!(sniff_tarball_format(&bz2), Some(TarballFormat::Bzip2));
+        } else {
+            std::fs::write(&bz2, b"BZh9\x31\x41\x59\x26\x53\x59").unwrap();
+            assert_eq!(sniff_tarball_format(&bz2), Some(TarballFormat::Bzip2));
+        }
+
+        let tar = dir.path().join("f.tar");
+        write_plain_tar_fixture(&tar, &[("a.txt", "A")]);
+        assert_eq!(sniff_tarball_format(&tar), Some(TarballFormat::PlainTar));
+
+        // Garbage (a zip header, notably) matches nothing.
+        let garbage = dir.path().join("garbage.bin");
+        std::fs::write(&garbage, b"PK\x03\x04 definitely not a tarball").unwrap();
+        assert_eq!(sniff_tarball_format(&garbage), None);
+        // Files too short to even hold a signature.
+        let tiny = dir.path().join("tiny.bin");
+        std::fs::write(&tiny, b"no").unwrap();
+        assert_eq!(sniff_tarball_format(&tiny), None);
+        // Missing files are simply unrecognized.
+        assert_eq!(sniff_tarball_format(&dir.path().join("missing")), None);
+    }
+
+    /// The extension fallback (for local user-provided files whose content
+    /// carries no recognizable magic) maps every supported name, including
+    /// the short spellings, case-insensitively.
+    #[test]
+    fn extension_fallback_mapping() {
+        fn p(name: &str) -> &Path {
+            Path::new(name)
+        }
+        assert_eq!(
+            format_from_extension(p("a.tar.gz")),
+            Some(TarballFormat::Gzip)
+        );
+        assert_eq!(format_from_extension(p("a.TGZ")), Some(TarballFormat::Gzip));
+        assert_eq!(
+            format_from_extension(p("a.tar.xz")),
+            Some(TarballFormat::Xz)
+        );
+        assert_eq!(format_from_extension(p("a.txz")), Some(TarballFormat::Xz));
+        assert_eq!(
+            format_from_extension(p("a.tar.bz2")),
+            Some(TarballFormat::Bzip2)
+        );
+        assert_eq!(
+            format_from_extension(p("a.tbz2")),
+            Some(TarballFormat::Bzip2)
+        );
+        assert_eq!(
+            format_from_extension(p("a.tar")),
+            Some(TarballFormat::PlainTar)
+        );
+        assert_eq!(format_from_extension(p("a.tar.zip")), None);
+        assert_eq!(format_from_extension(p("a")), None);
+    }
+
+    /// Regression (release-download origin): a downloaded stream lands in
+    /// an EXTENSIONLESS temporary file, which the extension-based check
+    /// always rejected ('does not look like a tarball'), silently falling
+    /// through to `git archive`. Valid gzip and plain-tar content must
+    /// round-trip through such a name.
+    #[test]
+    fn repack_sniffs_extensionless_downloaded_tarballs() {
+        let dir = tempfile::tempdir().unwrap();
+        let tree = dir.path().join("mytool");
+        std::fs::create_dir_all(&tree).unwrap();
+
+        // A gzip stream, like a codeload tar.gz download.
+        let source = dir.path().join("pkh-orig-1234");
+        write_gz_fixture(&source, &[("src/main.rs", "hi\n")]);
+        let dest = repack_tarball_file(&source, "mytool", "1.0.0", &tree).unwrap();
+        assert!(tarball_names(&dest).contains(&"mytool-1.0.0/src/main.rs".to_string()));
+
+        // A plain tar stream, also extensionless.
+        let source = dir.path().join("pkh-orig-5678");
+        write_plain_tar_fixture(&source, &[("README", "readme\n")]);
+        let dest = repack_tarball_file(&source, "mytool", "1.0.1", &tree).unwrap();
+        assert!(tarball_names(&dest).contains(&"mytool-1.0.1/README".to_string()));
+    }
+
+    /// The sniffed decompression route also covers xz (and bzip2, below):
+    /// a valid xz stream round-trips through an extensionless name.
+    #[test]
+    fn repack_sniffs_xz_tarballs() {
+        let dir = tempfile::tempdir().unwrap();
+        let tree = dir.path().join("mytool");
+        std::fs::create_dir_all(&tree).unwrap();
+
+        let source = dir.path().join("pkh-orig-xz");
+        write_xz_fixture(&source, &[("x.txt", "X\n")]);
+        let dest = repack_tarball_file(&source, "mytool", "1.1.0", &tree).unwrap();
+        assert!(tarball_names(&dest).contains(&"mytool-1.1.0/x.txt".to_string()));
+    }
+
+    /// bzip2 content is sniffed from its "BZh" header and still
+    /// decompressed through the host `bzip2` binary, extensionless or not.
+    #[test]
+    fn repack_sniffs_bzip2_tarballs() {
+        if !std::process::Command::new("bzip2")
+            .arg("--version")
+            .output()
+            .map(|o| o.status.success())
+            .unwrap_or(false)
+        {
+            return;
+        }
+        let dir = tempfile::tempdir().unwrap();
+        let tree = dir.path().join("mytool");
+        std::fs::create_dir_all(&tree).unwrap();
+
+        let source = dir.path().join("pkh-orig-bz2");
+        assert!(write_bzip2_fixture(&source, &[("b.txt", "B\n")]));
+        let dest = repack_tarball_file(&source, "mytool", "1.2.0", &tree).unwrap();
+        assert!(tarball_names(&dest).contains(&"mytool-1.2.0/b.txt".to_string()));
+    }
+
+    /// Content sniffing takes precedence over the name: a gzip stream
+    /// misnamed '.tar.xz' is decoded as gzip, not xz.
+    #[test]
+    fn repack_content_sniffing_beats_the_extension() {
+        let dir = tempfile::tempdir().unwrap();
+        let tree = dir.path().join("mytool");
+        std::fs::create_dir_all(&tree).unwrap();
+
+        let source = dir.path().join("misnamed.tar.xz");
+        write_gz_fixture(&source, &[("f.txt", "x\n")]);
+        let dest = repack_tarball_file(&source, "mytool", "1.3.0", &tree).unwrap();
+        assert!(tarball_names(&dest).contains(&"mytool-1.3.0/f.txt".to_string()));
+    }
+
+    /// Unsniffable content is rejected with a clear error whether or not
+    /// the name carries an extension — a download can only fail on
+    /// unrecognized CONTENT, never on a missing extension.
+    #[test]
+    fn repack_rejects_unsniffable_content() {
+        let dir = tempfile::tempdir().unwrap();
+        let tree = dir.path().join("mytool");
+        std::fs::create_dir_all(&tree).unwrap();
+
+        // Extensionless, like a downloaded temp file.
+        let source = dir.path().join("pkh-orig-0000");
+        std::fs::write(&source, b"PK\x03\x04 zip data").unwrap();
+        let err = repack_tarball_file(&source, "mytool", "9.9.9", &tree).unwrap_err();
+        assert!(
+            err.to_string().contains("does not look like a tarball"),
+            "{err}"
+        );
+        // No half-written tarball is left behind.
+        assert!(!dir.path().join("mytool_9.9.9.orig.tar.xz").exists());
+    }
+
     #[test]
     fn repack_preserves_the_exec_bit() {
         let dir = tempfile::tempdir().unwrap();