diff --git a/src/new/orig.rs b/src/new/orig.rs index 77da61e..5b8ef06 100644 --- a/src/new/orig.rs +++ b/src/new/orig.rs @@ -363,10 +363,89 @@ fn download_to_temp(url: &str) -> Result> { Ok(temp) } +/// Container/compression format of a tarball, detected from its content +/// or, as a fallback, from its name. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TarballFormat { + /// gzip stream (`1f 8b`), decoded natively. + Gzip, + /// xz stream (`fd 37 7a 58 5a 00`), decoded natively. + Xz, + /// bzip2 stream (`42 5a 68`, "BZh"), decompressed through the host + /// `bzip2` binary. + Bzip2, + /// Uncompressed tar (`ustar` at offset 257). + PlainTar, +} + +/// Sniff the format of the tarball at `path` from its leading magic bytes +/// rather than its name: gzip (`1f 8b`), xz (`fd 37 7a 58 5a 00`), bzip2 +/// (`"BZh"`) and plain tar (`ustar` at offset 257) are recognized. +/// Returns `None` when no signature matches (or the file cannot be read). +fn sniff_tarball_format(path: &Path) -> Option { + // 257 + len("ustar"): the longest signature lives at that offset. + let mut head = [0u8; 262]; + let mut file = std::fs::File::open(path).ok()?; + let mut filled = 0; + while filled < head.len() { + match file.read(&mut head[filled..]) { + Ok(0) => break, + Ok(n) => filled += n, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Err(_) => return None, + } + } + if head.starts_with(&[0x1f, 0x8b]) { + Some(TarballFormat::Gzip) + } else if head.starts_with(&[0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00]) { + Some(TarballFormat::Xz) + } else if head.starts_with(b"BZh") { + Some(TarballFormat::Bzip2) + } else if filled == head.len() && head[257..].starts_with(b"ustar") { + Some(TarballFormat::PlainTar) + } else { + None + } +} + +/// Map a tarball file NAME to its format — the fallback for local +/// user-provided files whose content carries no recognizable magic +/// signature (downloads are always sniffed first and never rejected for a +/// missing extension). Recognizes `.tar`, `.tar.gz`/`.tgz`, +/// `.tar.xz`/`.txz` and `.tar.bz2`/`.tbz2` (case-insensitively). +fn format_from_extension(path: &Path) -> Option { + let full = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default() + .to_ascii_lowercase(); + let extension = path + .extension() + .and_then(|ext| ext.to_str()) + .unwrap_or_default() + .to_ascii_lowercase(); + if full.ends_with(".tar.gz") || full.ends_with(".tgz") { + Some(TarballFormat::Gzip) + } else if full.ends_with(".tar.xz") || full.ends_with(".txz") { + Some(TarballFormat::Xz) + } else if full.ends_with(".tar.bz2") || full.ends_with(".tbz2") { + Some(TarballFormat::Bzip2) + } else if extension == "tar" { + Some(TarballFormat::PlainTar) + } else { + None + } +} + /// Open `path` as a (possibly compressed) tar stream and repack it under /// the `-/` prefix into `../_.orig.tar.xz`. -/// `.tar.bz2`/`.tbz2` inputs are decompressed through the host `bzip2` -/// binary (pkh carries no bzip2 codec); gz and xz are decoded natively. +/// The decompression route is decided by the file's CONTENT (magic bytes), +/// not its name: downloaded streams land in an extensionless temporary +/// file and must not be rejected for a missing extension. Only local +/// user-provided files whose content carries no recognizable signature +/// fall back to the file-name extension. bzip2 content is decompressed +/// through the host `bzip2` binary (pkh carries no bzip2 codec); gz and +/// xz are decoded natively. fn repack_tarball_file( path: &Path, name: &str, @@ -387,49 +466,44 @@ fn repack_tarball_file( .into()); } - let extension = path - .extension() - .and_then(|ext| ext.to_str()) - .unwrap_or_default() - .to_ascii_lowercase(); - let full = path - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default() - .to_ascii_lowercase(); + // Content decides the decompression route; the name is only a fallback + // for local files with no recognizable magic (an unrecognized download + // is rejected on its content, never on a missing extension). + let format = sniff_tarball_format(path) + .or_else(|| format_from_extension(path)) + .ok_or_else(|| { + format!( + "'{}' does not look like a tarball: unrecognized content \ + (expected gzip, xz, bzip2 or plain tar)", + path.display() + ) + })?; + log::debug!("Tarball '{}' detected as {format:?}", path.display()); let mut bzip2_child: Option = None; - let reader: Box = if full.ends_with(".tar.gz") || full.ends_with(".tgz") { - Box::new(flate2::read::GzDecoder::new(std::fs::File::open(path)?)) - } else if full.ends_with(".tar.xz") || full.ends_with(".txz") { - Box::new(xz2::read::XzDecoder::new(std::fs::File::open(path)?)) - } else if full.ends_with(".tar.bz2") || full.ends_with(".tbz2") { - let mut child = Command::new("bzip2") - .arg("-dc") - .arg(path) - .stdout(Stdio::piped()) - .spawn() - .map_err(|e| { - format!( - "'.tar.bz2' tarballs need the bzip2 binary on PATH to be \ - repacked: {e}" - ) - })?; - let stdout = child - .stdout - .take() - .ok_or_else(|| "bzip2 produced no output".to_string())?; - bzip2_child = Some(child); - Box::new(stdout) - } else if extension == "tar" { - Box::new(std::fs::File::open(path)?) - } else { - return Err(format!( - "'{}' does not look like a tarball: expected .tar, .tar.gz, \ - .tgz, .tar.bz2, .tbz2 or .tar.xz", - path.display() - ) - .into()); + let reader: Box = match format { + TarballFormat::Gzip => Box::new(flate2::read::GzDecoder::new(std::fs::File::open(path)?)), + TarballFormat::Xz => Box::new(xz2::read::XzDecoder::new(std::fs::File::open(path)?)), + TarballFormat::Bzip2 => { + let mut child = Command::new("bzip2") + .arg("-dc") + .arg(path) + .stdout(Stdio::piped()) + .spawn() + .map_err(|e| { + format!( + "'.tar.bz2' tarballs need the bzip2 binary on PATH to be \ + repacked: {e}" + ) + })?; + let stdout = child + .stdout + .take() + .ok_or_else(|| "bzip2 produced no output".to_string())?; + bzip2_child = Some(child); + Box::new(stdout) + } + TarballFormat::PlainTar => Box::new(std::fs::File::open(path)?), }; let result = repack_tar_stream(reader, name, upstream_version, &dest); @@ -543,12 +617,12 @@ mod tests { .collect() } - /// Build a gz tarball with the given entries (path → contents), under - /// the `oldpkg-1.0/` top-level directory. - fn write_gz_fixture(path: &Path, entries: &[(&str, &str)]) { - let file = std::fs::File::create(path).unwrap(); - let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast()); - let mut builder = tar::Builder::new(encoder); + /// Append the fixture entries (plus the `oldpkg-1.0/` top-level + /// directory) to any tar builder. + fn append_fixture_entries( + builder: &mut tar::Builder, + entries: &[(&str, &str)], + ) { let mut header = tar::Header::new_gnu(); header.set_entry_type(tar::EntryType::Directory); header.set_size(0); @@ -570,9 +644,62 @@ mod tests { ) .unwrap(); } + } + + /// Build a gz tarball with the given entries (path → contents), under + /// the `oldpkg-1.0/` top-level directory. + fn write_gz_fixture(path: &Path, entries: &[(&str, &str)]) { + let file = std::fs::File::create(path).unwrap(); + let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast()); + let mut builder = tar::Builder::new(encoder); + append_fixture_entries(&mut builder, entries); builder.into_inner().unwrap(); } + /// Build an uncompressed tar with the given entries, under the + /// `oldpkg-1.0/` top-level directory. + fn write_plain_tar_fixture(path: &Path, entries: &[(&str, &str)]) { + let mut builder = tar::Builder::new(std::fs::File::create(path).unwrap()); + append_fixture_entries(&mut builder, entries); + builder.finish().unwrap(); + } + + /// Build an xz tarball with the given entries, under the `oldpkg-1.0/` + /// top-level directory. + fn write_xz_fixture(path: &Path, entries: &[(&str, &str)]) { + let file = std::fs::File::create(path).unwrap(); + let encoder = xz2::write::XzEncoder::new(file, 1); + let mut builder = tar::Builder::new(encoder); + append_fixture_entries(&mut builder, entries); + builder.finish().unwrap(); + } + + /// Build a bzip2 tarball with the given entries by piping a plain tar + /// through the host `bzip2` binary (pkh carries no bzip2 codec). + /// Returns false when the binary is unavailable; callers skip the + /// bzip2-specific assertions then. + fn write_bzip2_fixture(path: &Path, entries: &[(&str, &str)]) -> bool { + if std::process::Command::new("bzip2") + .arg("--version") + .output() + .is_err() + { + return false; + } + let plain_path = path.with_extension("tar"); + write_plain_tar_fixture(&plain_path, entries); + let compressed = std::fs::File::create(path).unwrap(); + let ok = std::process::Command::new("bzip2") + .arg("-zc") + .arg(&plain_path) + .stdout(Stdio::from(compressed)) + .status() + .map(|status| status.success()) + .unwrap_or(false); + let _ = std::fs::remove_file(&plain_path); + ok + } + #[test] fn repack_rewrites_the_top_level_prefix() { let dir = tempfile::tempdir().unwrap(); @@ -684,6 +811,177 @@ mod tests { assert!(tarball_names(&dest).contains(&"mytool-2.1/g.txt".to_string())); } + /// Every tarball format is recognized from its real content alone: + /// gzip (`1f 8b`), xz (`fd 37 7a 58 5a 00`), bzip2 (`"BZh"`) and + /// plain tar (`ustar` at offset 257). Garbage and truncated files + /// match nothing. + #[test] + fn sniff_detects_every_magic_signature() { + let dir = tempfile::tempdir().unwrap(); + + let gz = dir.path().join("f.tar.gz"); + write_gz_fixture(&gz, &[("a.txt", "A")]); + assert_eq!(sniff_tarball_format(&gz), Some(TarballFormat::Gzip)); + + let xz = dir.path().join("f.tar.xz"); + write_xz_fixture(&xz, &[("a.txt", "A")]); + assert_eq!(sniff_tarball_format(&xz), Some(TarballFormat::Xz)); + + // Real bzip2 bytes when the host binary exists, else the raw + // "BZh" header the sniffer keys on. + let bz2 = dir.path().join("f.tar.bz2"); + if write_bzip2_fixture(&bz2, &[("a.txt", "A")]) { + assert_eq!(sniff_tarball_format(&bz2), Some(TarballFormat::Bzip2)); + } else { + std::fs::write(&bz2, b"BZh9\x31\x41\x59\x26\x53\x59").unwrap(); + assert_eq!(sniff_tarball_format(&bz2), Some(TarballFormat::Bzip2)); + } + + let tar = dir.path().join("f.tar"); + write_plain_tar_fixture(&tar, &[("a.txt", "A")]); + assert_eq!(sniff_tarball_format(&tar), Some(TarballFormat::PlainTar)); + + // Garbage (a zip header, notably) matches nothing. + let garbage = dir.path().join("garbage.bin"); + std::fs::write(&garbage, b"PK\x03\x04 definitely not a tarball").unwrap(); + assert_eq!(sniff_tarball_format(&garbage), None); + // Files too short to even hold a signature. + let tiny = dir.path().join("tiny.bin"); + std::fs::write(&tiny, b"no").unwrap(); + assert_eq!(sniff_tarball_format(&tiny), None); + // Missing files are simply unrecognized. + assert_eq!(sniff_tarball_format(&dir.path().join("missing")), None); + } + + /// The extension fallback (for local user-provided files whose content + /// carries no recognizable magic) maps every supported name, including + /// the short spellings, case-insensitively. + #[test] + fn extension_fallback_mapping() { + fn p(name: &str) -> &Path { + Path::new(name) + } + assert_eq!( + format_from_extension(p("a.tar.gz")), + Some(TarballFormat::Gzip) + ); + assert_eq!(format_from_extension(p("a.TGZ")), Some(TarballFormat::Gzip)); + assert_eq!( + format_from_extension(p("a.tar.xz")), + Some(TarballFormat::Xz) + ); + assert_eq!(format_from_extension(p("a.txz")), Some(TarballFormat::Xz)); + assert_eq!( + format_from_extension(p("a.tar.bz2")), + Some(TarballFormat::Bzip2) + ); + assert_eq!( + format_from_extension(p("a.tbz2")), + Some(TarballFormat::Bzip2) + ); + assert_eq!( + format_from_extension(p("a.tar")), + Some(TarballFormat::PlainTar) + ); + assert_eq!(format_from_extension(p("a.tar.zip")), None); + assert_eq!(format_from_extension(p("a")), None); + } + + /// Regression (release-download origin): a downloaded stream lands in + /// an EXTENSIONLESS temporary file, which the extension-based check + /// always rejected ('does not look like a tarball'), silently falling + /// through to `git archive`. Valid gzip and plain-tar content must + /// round-trip through such a name. + #[test] + fn repack_sniffs_extensionless_downloaded_tarballs() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(&tree).unwrap(); + + // A gzip stream, like a codeload tar.gz download. + let source = dir.path().join("pkh-orig-1234"); + write_gz_fixture(&source, &[("src/main.rs", "hi\n")]); + let dest = repack_tarball_file(&source, "mytool", "1.0.0", &tree).unwrap(); + assert!(tarball_names(&dest).contains(&"mytool-1.0.0/src/main.rs".to_string())); + + // A plain tar stream, also extensionless. + let source = dir.path().join("pkh-orig-5678"); + write_plain_tar_fixture(&source, &[("README", "readme\n")]); + let dest = repack_tarball_file(&source, "mytool", "1.0.1", &tree).unwrap(); + assert!(tarball_names(&dest).contains(&"mytool-1.0.1/README".to_string())); + } + + /// The sniffed decompression route also covers xz (and bzip2, below): + /// a valid xz stream round-trips through an extensionless name. + #[test] + fn repack_sniffs_xz_tarballs() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(&tree).unwrap(); + + let source = dir.path().join("pkh-orig-xz"); + write_xz_fixture(&source, &[("x.txt", "X\n")]); + let dest = repack_tarball_file(&source, "mytool", "1.1.0", &tree).unwrap(); + assert!(tarball_names(&dest).contains(&"mytool-1.1.0/x.txt".to_string())); + } + + /// bzip2 content is sniffed from its "BZh" header and still + /// decompressed through the host `bzip2` binary, extensionless or not. + #[test] + fn repack_sniffs_bzip2_tarballs() { + if !std::process::Command::new("bzip2") + .arg("--version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) + { + return; + } + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(&tree).unwrap(); + + let source = dir.path().join("pkh-orig-bz2"); + assert!(write_bzip2_fixture(&source, &[("b.txt", "B\n")])); + let dest = repack_tarball_file(&source, "mytool", "1.2.0", &tree).unwrap(); + assert!(tarball_names(&dest).contains(&"mytool-1.2.0/b.txt".to_string())); + } + + /// Content sniffing takes precedence over the name: a gzip stream + /// misnamed '.tar.xz' is decoded as gzip, not xz. + #[test] + fn repack_content_sniffing_beats_the_extension() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(&tree).unwrap(); + + let source = dir.path().join("misnamed.tar.xz"); + write_gz_fixture(&source, &[("f.txt", "x\n")]); + let dest = repack_tarball_file(&source, "mytool", "1.3.0", &tree).unwrap(); + assert!(tarball_names(&dest).contains(&"mytool-1.3.0/f.txt".to_string())); + } + + /// Unsniffable content is rejected with a clear error whether or not + /// the name carries an extension — a download can only fail on + /// unrecognized CONTENT, never on a missing extension. + #[test] + fn repack_rejects_unsniffable_content() { + let dir = tempfile::tempdir().unwrap(); + let tree = dir.path().join("mytool"); + std::fs::create_dir_all(&tree).unwrap(); + + // Extensionless, like a downloaded temp file. + let source = dir.path().join("pkh-orig-0000"); + std::fs::write(&source, b"PK\x03\x04 zip data").unwrap(); + let err = repack_tarball_file(&source, "mytool", "9.9.9", &tree).unwrap_err(); + assert!( + err.to_string().contains("does not look like a tarball"), + "{err}" + ); + // No half-written tarball is left behind. + assert!(!dir.path().join("mytool_9.9.9.orig.tar.xz").exists()); + } + #[test] fn repack_preserves_the_exec_bit() { let dir = tempfile::tempdir().unwrap();