vhaudiquet/pkh
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

new: detect downloaded tarballs by magic bytes, not extension

Browse Source
This commit is contained in:
vhaudiquet
2026-09-17 02:15:23 +02:00
parent f055b70281
commit 05e7c55d32
1 changed files with 347 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/new/orig.rs
+347 -49
View File
@@ -363,10 +363,89 @@ fn download_to_temp(url: &str) -> Result<PathBuf, Box<dyn std::error::Error>> {
Ok(temp)
}
/// Container/compression format of a tarball, detected from its content
/// or, as a fallback, from its name.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum TarballFormat {
/// gzip stream (`1f 8b`), decoded natively.
Gzip,
/// xz stream (`fd 37 7a 58 5a 00`), decoded natively.
Xz,
/// bzip2 stream (`42 5a 68`, "BZh"), decompressed through the host
/// `bzip2` binary.
Bzip2,
/// Uncompressed tar (`ustar` at offset 257).
PlainTar,
}
/// Sniff the format of the tarball at `path` from its leading magic bytes
/// rather than its name: gzip (`1f 8b`), xz (`fd 37 7a 58 5a 00`), bzip2
/// (`"BZh"`) and plain tar (`ustar` at offset 257) are recognized.
/// Returns `None` when no signature matches (or the file cannot be read).
fn sniff_tarball_format(path: &Path) -> Option<TarballFormat> {
// 257 + len("ustar"): the longest signature lives at that offset.
let mut head = [0u8; 262];
let mut file = std::fs::File::open(path).ok()?;
let mut filled = 0;
while filled < head.len() {
match file.read(&mut head[filled..]) {
Ok(0) => break,
Ok(n) => filled += n,
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
Err(_) => return None,
}
}
if head.starts_with(&[0x1f, 0x8b]) {
Some(TarballFormat::Gzip)
} else if head.starts_with(&[0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00]) {
Some(TarballFormat::Xz)
} else if head.starts_with(b"BZh") {
Some(TarballFormat::Bzip2)
} else if filled == head.len() && head[257..].starts_with(b"ustar") {
Some(TarballFormat::PlainTar)
} else {
None
}
}
/// Map a tarball file NAME to its format — the fallback for local
/// user-provided files whose content carries no recognizable magic
/// signature (downloads are always sniffed first and never rejected for a
/// missing extension). Recognizes `.tar`, `.tar.gz`/`.tgz`,
/// `.tar.xz`/`.txz` and `.tar.bz2`/`.tbz2` (case-insensitively).
fn format_from_extension(path: &Path) -> Option<TarballFormat> {
let full = path
.file_name()
.and_then(|name| name.to_str())
.unwrap_or_default()
.to_ascii_lowercase();
let extension = path
.extension()
.and_then(|ext| ext.to_str())
.unwrap_or_default()
.to_ascii_lowercase();
if full.ends_with(".tar.gz") || full.ends_with(".tgz") {
Some(TarballFormat::Gzip)
} else if full.ends_with(".tar.xz") || full.ends_with(".txz") {
Some(TarballFormat::Xz)
} else if full.ends_with(".tar.bz2") || full.ends_with(".tbz2") {
Some(TarballFormat::Bzip2)
} else if extension == "tar" {
Some(TarballFormat::PlainTar)
} else {
None
}
}
/// Open `path` as a (possibly compressed) tar stream and repack it under
/// the `<name>-<uver>/` prefix into `../<name>_<uver>.orig.tar.xz`.
/// `.tar.bz2`/`.tbz2` inputs are decompressed through the host `bzip2`
/// binary (pkh carries no bzip2 codec); gz and xz are decoded natively.
/// The decompression route is decided by the file's CONTENT (magic bytes),
/// not its name: downloaded streams land in an extensionless temporary
/// file and must not be rejected for a missing extension. Only local
/// user-provided files whose content carries no recognizable signature
/// fall back to the file-name extension. bzip2 content is decompressed
/// through the host `bzip2` binary (pkh carries no bzip2 codec); gz and
/// xz are decoded natively.
fn repack_tarball_file(
path: &Path,
name: &str,
@@ -387,49 +466,44 @@ fn repack_tarball_file(
.into());
}
let extension = path
.extension()
.and_then(|ext| ext.to_str())
.unwrap_or_default()
.to_ascii_lowercase();
let full = path
.file_name()
.and_then(|name| name.to_str())
.unwrap_or_default()
.to_ascii_lowercase();
// Content decides the decompression route; the name is only a fallback
// for local files with no recognizable magic (an unrecognized download
// is rejected on its content, never on a missing extension).
let format = sniff_tarball_format(path)
.or_else(|| format_from_extension(path))
.ok_or_else(|| {
format!(
"'{}' does not look like a tarball: unrecognized content \
(expected gzip, xz, bzip2 or plain tar)",
path.display()
)
})?;
log::debug!("Tarball '{}' detected as {format:?}", path.display());
let mut bzip2_child: Option<std::process::Child> = None;
let reader: Box<dyn Read> = if full.ends_with(".tar.gz") || full.ends_with(".tgz") {
Box::new(flate2::read::GzDecoder::new(std::fs::File::open(path)?))
} else if full.ends_with(".tar.xz") || full.ends_with(".txz") {
Box::new(xz2::read::XzDecoder::new(std::fs::File::open(path)?))
} else if full.ends_with(".tar.bz2") || full.ends_with(".tbz2") {
let mut child = Command::new("bzip2")
.arg("-dc")
.arg(path)
.stdout(Stdio::piped())
.spawn()
.map_err(|e| {
format!(
"'.tar.bz2' tarballs need the bzip2 binary on PATH to be \
repacked: {e}"
)
})?;
let stdout = child
.stdout
.take()
.ok_or_else(|| "bzip2 produced no output".to_string())?;
bzip2_child = Some(child);
Box::new(stdout)
} else if extension == "tar" {
Box::new(std::fs::File::open(path)?)
} else {
return Err(format!(
"'{}' does not look like a tarball: expected .tar, .tar.gz, \
.tgz, .tar.bz2, .tbz2 or .tar.xz",
path.display()
)
.into());
let reader: Box<dyn Read> = match format {
TarballFormat::Gzip => Box::new(flate2::read::GzDecoder::new(std::fs::File::open(path)?)),
TarballFormat::Xz => Box::new(xz2::read::XzDecoder::new(std::fs::File::open(path)?)),
TarballFormat::Bzip2 => {
let mut child = Command::new("bzip2")
.arg("-dc")
.arg(path)
.stdout(Stdio::piped())
.spawn()
.map_err(|e| {
format!(
"'.tar.bz2' tarballs need the bzip2 binary on PATH to be \
repacked: {e}"
)
})?;
let stdout = child
.stdout
.take()
.ok_or_else(|| "bzip2 produced no output".to_string())?;
bzip2_child = Some(child);
Box::new(stdout)
}
TarballFormat::PlainTar => Box::new(std::fs::File::open(path)?),
};
let result = repack_tar_stream(reader, name, upstream_version, &dest);
@@ -543,12 +617,12 @@ mod tests {
.collect()
}
/// Build a gz tarball with the given entries (path → contents), under
/// the `oldpkg-1.0/` top-level directory.
fn write_gz_fixture(path: &Path, entries: &[(&str, &str)]) {
let file = std::fs::File::create(path).unwrap();
let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
let mut builder = tar::Builder::new(encoder);
/// Append the fixture entries (plus the `oldpkg-1.0/` top-level
/// directory) to any tar builder.
fn append_fixture_entries<W: std::io::Write>(
builder: &mut tar::Builder<W>,
entries: &[(&str, &str)],
) {
let mut header = tar::Header::new_gnu();
header.set_entry_type(tar::EntryType::Directory);
header.set_size(0);
@@ -570,9 +644,62 @@ mod tests {
)
.unwrap();
}
}
/// Build a gz tarball with the given entries (path → contents), under
/// the `oldpkg-1.0/` top-level directory.
fn write_gz_fixture(path: &Path, entries: &[(&str, &str)]) {
let file = std::fs::File::create(path).unwrap();
let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
let mut builder = tar::Builder::new(encoder);
append_fixture_entries(&mut builder, entries);
builder.into_inner().unwrap();
}
/// Build an uncompressed tar with the given entries, under the
/// `oldpkg-1.0/` top-level directory.
fn write_plain_tar_fixture(path: &Path, entries: &[(&str, &str)]) {
let mut builder = tar::Builder::new(std::fs::File::create(path).unwrap());
append_fixture_entries(&mut builder, entries);
builder.finish().unwrap();
}
/// Build an xz tarball with the given entries, under the `oldpkg-1.0/`
/// top-level directory.
fn write_xz_fixture(path: &Path, entries: &[(&str, &str)]) {
let file = std::fs::File::create(path).unwrap();
let encoder = xz2::write::XzEncoder::new(file, 1);
let mut builder = tar::Builder::new(encoder);
append_fixture_entries(&mut builder, entries);
builder.finish().unwrap();
}
/// Build a bzip2 tarball with the given entries by piping a plain tar
/// through the host `bzip2` binary (pkh carries no bzip2 codec).
/// Returns false when the binary is unavailable; callers skip the
/// bzip2-specific assertions then.
fn write_bzip2_fixture(path: &Path, entries: &[(&str, &str)]) -> bool {
if std::process::Command::new("bzip2")
.arg("--version")
.output()
.is_err()
{
return false;
}
let plain_path = path.with_extension("tar");
write_plain_tar_fixture(&plain_path, entries);
let compressed = std::fs::File::create(path).unwrap();
let ok = std::process::Command::new("bzip2")
.arg("-zc")
.arg(&plain_path)
.stdout(Stdio::from(compressed))
.status()
.map(|status| status.success())
.unwrap_or(false);
let _ = std::fs::remove_file(&plain_path);
ok
}
#[test]
fn repack_rewrites_the_top_level_prefix() {
let dir = tempfile::tempdir().unwrap();
@@ -684,6 +811,177 @@ mod tests {
assert!(tarball_names(&dest).contains(&"mytool-2.1/g.txt".to_string()));
}
/// Every tarball format is recognized from its real content alone:
/// gzip (`1f 8b`), xz (`fd 37 7a 58 5a 00`), bzip2 (`"BZh"`) and
/// plain tar (`ustar` at offset 257). Garbage and truncated files
/// match nothing.
#[test]
fn sniff_detects_every_magic_signature() {
let dir = tempfile::tempdir().unwrap();
let gz = dir.path().join("f.tar.gz");
write_gz_fixture(&gz, &[("a.txt", "A")]);
assert_eq!(sniff_tarball_format(&gz), Some(TarballFormat::Gzip));
let xz = dir.path().join("f.tar.xz");
write_xz_fixture(&xz, &[("a.txt", "A")]);
assert_eq!(sniff_tarball_format(&xz), Some(TarballFormat::Xz));
// Real bzip2 bytes when the host binary exists, else the raw
// "BZh" header the sniffer keys on.
let bz2 = dir.path().join("f.tar.bz2");
if write_bzip2_fixture(&bz2, &[("a.txt", "A")]) {
assert_eq!(sniff_tarball_format(&bz2), Some(TarballFormat::Bzip2));
} else {
std::fs::write(&bz2, b"BZh9\x31\x41\x59\x26\x53\x59").unwrap();
assert_eq!(sniff_tarball_format(&bz2), Some(TarballFormat::Bzip2));
}
let tar = dir.path().join("f.tar");
write_plain_tar_fixture(&tar, &[("a.txt", "A")]);
assert_eq!(sniff_tarball_format(&tar), Some(TarballFormat::PlainTar));
// Garbage (a zip header, notably) matches nothing.
let garbage = dir.path().join("garbage.bin");
std::fs::write(&garbage, b"PK\x03\x04 definitely not a tarball").unwrap();
assert_eq!(sniff_tarball_format(&garbage), None);
// Files too short to even hold a signature.
let tiny = dir.path().join("tiny.bin");
std::fs::write(&tiny, b"no").unwrap();
assert_eq!(sniff_tarball_format(&tiny), None);
// Missing files are simply unrecognized.
assert_eq!(sniff_tarball_format(&dir.path().join("missing")), None);
}
/// The extension fallback (for local user-provided files whose content
/// carries no recognizable magic) maps every supported name, including
/// the short spellings, case-insensitively.
#[test]
fn extension_fallback_mapping() {
fn p(name: &str) -> &Path {
Path::new(name)
}
assert_eq!(
format_from_extension(p("a.tar.gz")),
Some(TarballFormat::Gzip)
);
assert_eq!(format_from_extension(p("a.TGZ")), Some(TarballFormat::Gzip));
assert_eq!(
format_from_extension(p("a.tar.xz")),
Some(TarballFormat::Xz)
);
assert_eq!(format_from_extension(p("a.txz")), Some(TarballFormat::Xz));
assert_eq!(
format_from_extension(p("a.tar.bz2")),
Some(TarballFormat::Bzip2)
);
assert_eq!(
format_from_extension(p("a.tbz2")),
Some(TarballFormat::Bzip2)
);
assert_eq!(
format_from_extension(p("a.tar")),
Some(TarballFormat::PlainTar)
);
assert_eq!(format_from_extension(p("a.tar.zip")), None);
assert_eq!(format_from_extension(p("a")), None);
}
/// Regression (release-download origin): a downloaded stream lands in
/// an EXTENSIONLESS temporary file, which the extension-based check
/// always rejected ('does not look like a tarball'), silently falling
/// through to `git archive`. Valid gzip and plain-tar content must
/// round-trip through such a name.
#[test]
fn repack_sniffs_extensionless_downloaded_tarballs() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
// A gzip stream, like a codeload tar.gz download.
let source = dir.path().join("pkh-orig-1234");
write_gz_fixture(&source, &[("src/main.rs", "hi\n")]);
let dest = repack_tarball_file(&source, "mytool", "1.0.0", &tree).unwrap();
assert!(tarball_names(&dest).contains(&"mytool-1.0.0/src/main.rs".to_string()));
// A plain tar stream, also extensionless.
let source = dir.path().join("pkh-orig-5678");
write_plain_tar_fixture(&source, &[("README", "readme\n")]);
let dest = repack_tarball_file(&source, "mytool", "1.0.1", &tree).unwrap();
assert!(tarball_names(&dest).contains(&"mytool-1.0.1/README".to_string()));
}
/// The sniffed decompression route also covers xz (and bzip2, below):
/// a valid xz stream round-trips through an extensionless name.
#[test]
fn repack_sniffs_xz_tarballs() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
let source = dir.path().join("pkh-orig-xz");
write_xz_fixture(&source, &[("x.txt", "X\n")]);
let dest = repack_tarball_file(&source, "mytool", "1.1.0", &tree).unwrap();
assert!(tarball_names(&dest).contains(&"mytool-1.1.0/x.txt".to_string()));
}
/// bzip2 content is sniffed from its "BZh" header and still
/// decompressed through the host `bzip2` binary, extensionless or not.
#[test]
fn repack_sniffs_bzip2_tarballs() {
if !std::process::Command::new("bzip2")
.arg("--version")
.output()
.map(|o| o.status.success())
.unwrap_or(false)
{
return;
}
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
let source = dir.path().join("pkh-orig-bz2");
assert!(write_bzip2_fixture(&source, &[("b.txt", "B\n")]));
let dest = repack_tarball_file(&source, "mytool", "1.2.0", &tree).unwrap();
assert!(tarball_names(&dest).contains(&"mytool-1.2.0/b.txt".to_string()));
}
/// Content sniffing takes precedence over the name: a gzip stream
/// misnamed '.tar.xz' is decoded as gzip, not xz.
#[test]
fn repack_content_sniffing_beats_the_extension() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
let source = dir.path().join("misnamed.tar.xz");
write_gz_fixture(&source, &[("f.txt", "x\n")]);
let dest = repack_tarball_file(&source, "mytool", "1.3.0", &tree).unwrap();
assert!(tarball_names(&dest).contains(&"mytool-1.3.0/f.txt".to_string()));
}
/// Unsniffable content is rejected with a clear error whether or not
/// the name carries an extension — a download can only fail on
/// unrecognized CONTENT, never on a missing extension.
#[test]
fn repack_rejects_unsniffable_content() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
// Extensionless, like a downloaded temp file.
let source = dir.path().join("pkh-orig-0000");
std::fs::write(&source, b"PK\x03\x04 zip data").unwrap();
let err = repack_tarball_file(&source, "mytool", "9.9.9", &tree).unwrap_err();
assert!(
err.to_string().contains("does not look like a tarball"),
"{err}"
);
// No half-written tarball is left behind.
assert!(!dir.path().join("mytool_9.9.9.orig.tar.xz").exists());
}
#[test]
fn repack_preserves_the_exec_bit() {
let dir = tempfile::tempdir().unwrap();