Files
pkh/src/new/orig.rs
T
vhaudiquet eaf1b40369 new: stream orig downloads, reap children on every path
The release-tarball download capped the whole request at 30 s (large
tarballs on slow links always failed and fell through to worse origins)
and buffered the entire body in memory: keep a 10 s connect timeout
only and stream the body to the temp file. The bzip2 -dc child of a
failing repack was neither killed nor waited on (zombie + open pipe);
it is now reaped on both paths. git archive no longer pipes a stderr
nobody drains (a chatty git deadlocked the archive) and any failure
after the destination file was created removes the empty or partial
tarball.
2026-09-17 18:03:24 +02:00

1651 lines
65 KiB
Rust

//! Orig tarball creation for `pkh new`, one implementation per
//! [`OrigOrigin`]: working-tree snapshot, `git archive` of a release tag,
//! download of the forge release tarball, or repack of a user-provided
//! tarball — plus the dpkg upstream component tarball holding the vendored
//! Cargo dependencies (`<name>_<uver>.orig-vendor.tar.xz`).
//!
//! Whatever the origin, the tarball always lands at
//! `../<name>_<uver>.orig.tar.xz` with `<name>-<uver>/` as its single
//! top-level directory (what dpkg-source expects), and — for a vendored
//! rust package — always excludes the generated `vendor/` tree, which
//! travels in the component tarball instead and can be regenerated
//! independently of the upstream sources.
use std::ffi::OsString;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::Duration;
use xz2::write::XzEncoder;
use super::options::OrigOrigin;
use crate::debian::DebianVersion;
use crate::new::origin::Forge;
/// The upstream version dpkg names orig/component files after: the
/// changelog version's upstream part, with the epoch and the Debian
/// revision stripped (`1:0.14.0-1` → `0.14.0`).
///
/// dpkg-source globs `../<name>_<upstream>.orig.tar.xz` and
/// `../<name>_<upstream>.orig-<component>.tar.<ext>` — never the full
/// version — so every lookup or creation of those artifacts must derive
/// the name through here. Using `DebianVersion::no_epoch()` instead yields
/// `0.14.0-1` and silently misses the real component.
pub fn component_upstream_version(changelog_version: &DebianVersion) -> &str {
&changelog_version.upstream
}
/// Result of the orig creation: where the tarball landed and how it was
/// actually produced (the release download falls through to `git archive`
/// or a snapshot on failure, so the label can differ from the plan).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CreatedOrig {
/// Path of the written `<name>_<uver>.orig.tar.xz`.
pub path: PathBuf,
/// Human-readable description of the actual origin.
pub label: String,
}
/// Create the orig tarball of a quilt package according to `plan`.
///
/// `vendored_rust` marks a rust package whose tree carries a generated
/// `vendor/` directory: the snapshot origin then excludes it (the other
/// origins never contain it in the first place — upstream tags predate the
/// vendoring). Failures of an explicit user choice (`--orig-from path`)
/// are fatal; a failed release download is only a warning and falls
/// through to `git archive` of the tag, then to the snapshot.
pub fn create_orig(
tree: &Path,
name: &str,
upstream_version: &str,
plan: &OrigOrigin,
vendored_rust: bool,
) -> Result<CreatedOrig, Box<dyn std::error::Error>> {
match plan {
OrigOrigin::Snapshot => {
let path = super::debian::create_orig_tarball_excluding(
tree,
name,
upstream_version,
vendored_rust,
)?;
Ok(CreatedOrig {
path,
label: OrigOrigin::Snapshot.label(),
})
}
OrigOrigin::GitArchive { tag } => {
let path = git_archive_tarball(tree, tag, name, upstream_version)?;
Ok(CreatedOrig {
path,
label: OrigOrigin::GitArchive { tag: tag.clone() }.label(),
})
}
OrigOrigin::Release { tag, forge } => {
match download_release(forge, tag, name, upstream_version, tree) {
Ok(path) => Ok(CreatedOrig {
path,
label: OrigOrigin::Release {
tag: tag.clone(),
forge: forge.clone(),
}
.label(),
}),
Err(download_error) => {
log::warn!(
"The release tarball of {tag} could not be downloaded \
({download_error:#}); falling back to `git archive` \
of the tag, then to a working-tree snapshot"
);
match git_archive_tarball(tree, tag, name, upstream_version) {
Ok(path) => Ok(CreatedOrig {
path,
label: OrigOrigin::GitArchive { tag: tag.clone() }.label(),
}),
Err(archive_error) => {
log::warn!(
"`git archive` of {tag} failed too ({archive_error:#}); \
snapshotting the working tree instead"
);
let path = super::debian::create_orig_tarball_excluding(
tree,
name,
upstream_version,
vendored_rust,
)?;
Ok(CreatedOrig {
path,
label: OrigOrigin::Snapshot.label(),
})
}
}
}
}
}
OrigOrigin::Provided { source } => {
let path = fetch_and_repack(source, name, upstream_version, tree)?;
Ok(CreatedOrig {
path,
label: OrigOrigin::Provided {
source: source.clone(),
}
.label(),
})
}
}
}
/// Create `../<name>_<uver>.orig-vendor.tar.xz` holding the tree's
/// `vendor/` directory under a top-level `vendor/` path — the dpkg upstream
/// component dpkg-source unpacks back into the tree next to the main orig.
/// Refuses to overwrite an existing component (stale components are
/// removed by the caller, e.g. the re-vendoring retry of `pkh build`).
pub fn create_vendor_component(
tree: &Path,
name: &str,
upstream_version: &str,
) -> Result<PathBuf, Box<dyn std::error::Error>> {
let vendor = tree.join("vendor");
if !is_non_empty_dir(&vendor) {
return Err(format!(
"'{}' does not exist or is empty: nothing to put into the \
orig-vendor component",
vendor.display()
)
.into());
}
let component_path = vendor_component_path(tree, name, upstream_version).ok_or_else(|| {
format!(
"cannot determine the parent directory of '{}'",
tree.display()
)
})?;
if component_path.exists() {
return Err(format!(
"'{}' already exists: pkh new refuses to overwrite it. \
Remove the stale component first.",
component_path.display()
)
.into());
}
let file = std::fs::File::create(&component_path)?;
let encoder = XzEncoder::new(file, 6);
let mut builder = tar::Builder::new(encoder);
// Everything under vendor/ travels; no exclusions (vendored crates have
// no build leftovers) and no debian/ special case below the top level.
builder.append_dir("vendor", &vendor)?;
super::debian::append_tree(&mut builder, &vendor, "vendor", 0, &[], &[])?;
builder
.finish()
.map_err(|e| format!("failed to write '{}': {}", component_path.display(), e))?;
log::info!(
"Created vendored-dependencies component {}",
crate::ui::display_path(&component_path)
);
Ok(component_path)
}
/// Path of the dpkg upstream component holding `vendor/`, next to the tree.
pub fn vendor_component_path(tree: &Path, name: &str, upstream_version: &str) -> Option<PathBuf> {
tree.parent()
.map(|parent| parent.join(format!("{name}_{upstream_version}.orig-vendor.tar.xz")))
}
/// Whether `tree` carries a generated, non-empty `vendor/` directory.
pub fn has_vendored_dir(tree: &Path) -> bool {
is_non_empty_dir(&tree.join("vendor"))
}
fn is_non_empty_dir(path: &Path) -> bool {
path.is_dir() && std::fs::read_dir(path).is_ok_and(|mut entries| entries.next().is_some())
}
/// `git archive --format=tar --prefix=<name>-<uver>/ <tag>` compressed to
/// `../<name>_<uver>.orig.tar.xz`: offline and byte-deterministic.
fn git_archive_tarball(
repo: &Path,
tag: &str,
name: &str,
upstream_version: &str,
) -> Result<PathBuf, Box<dyn std::error::Error>> {
let dest = super::debian::orig_tarball_path(repo, name, upstream_version).ok_or_else(|| {
format!(
"cannot determine the parent directory of '{}'",
repo.display()
)
})?;
if dest.exists() {
return Err(format!(
"'{}' already exists: pkh new refuses to overwrite it.",
dest.display()
)
.into());
}
// Any failure after the destination file was created (spawn error,
// broken pipe, failed git) leaves behind an empty or half-written xz
// file that is not a valid tarball: it is removed before returning.
let outcome = (|| {
let file = std::fs::File::create(&dest)?;
let mut encoder = XzEncoder::new(file, 6);
// git's stderr is inherited, not piped: nothing here drains a piped
// stderr, and a chatty git filling the 64 KiB pipe buffer would
// deadlock the archive — its diagnostics belong on the terminal
// anyway, like every other child process in this module.
let mut child = Command::new("git")
.args([
"archive",
"--format=tar",
&format!("--prefix={name}-{upstream_version}/"),
tag,
])
.current_dir(repo)
.stdout(Stdio::piped())
.spawn()
.map_err(|e| format!("failed to run 'git archive': {e}"))?;
let mut stdout = child
.stdout
.take()
.ok_or_else(|| "git archive produced no output".to_string())?;
let copied = std::io::copy(&mut stdout, &mut encoder)
.map_err(|e| format!("cannot pipe git archive into '{}': {e}", dest.display()));
// The child is reaped whatever happened to the pipe, and a copy
// error only surfaces once it has been waited on. Closing our end
// first unblocks a git still writing into the pipe (SIGPIPE);
// otherwise `wait` could hang on it forever.
drop(stdout);
let status = child.wait()?;
copied?;
encoder
.finish()
.map_err(|e| format!("failed to write '{}': {e}", dest.display()))?;
Ok(status)
})();
match outcome {
Ok(status) if status.success() => {}
Ok(status) => {
let _ = std::fs::remove_file(&dest);
return Err(format!(
"'git archive --format=tar {}' failed with status: {status} \
(is HEAD exactly on the tag '{tag}'?)",
tag
)
.into());
}
Err(error) => {
let _ = std::fs::remove_file(&dest);
return Err(error);
}
}
log::info!(
"Created orig tarball from git archive of {tag}: {}",
crate::ui::display_path(&dest)
);
Ok(dest)
}
/// Download the release tarball of `tag` from `forge` (the user's choice of
/// this origin IS the network consent) and repack it to
/// `../<name>_<uver>.orig.tar.xz`. All of the forge's candidate URLs are
/// tried before failing.
fn download_release(
forge: &Forge,
tag: &str,
name: &str,
upstream_version: &str,
tree: &Path,
) -> Result<PathBuf, Box<dyn std::error::Error>> {
let urls = forge.release_tarball_urls(tag);
let mut last_error: Option<Box<dyn std::error::Error>> = None;
for url in &urls {
log::info!("Downloading the upstream release tarball from {url}");
match download_to_temp(url) {
Ok(temp) => {
let result = repack_tarball_file(&temp, name, upstream_version, tree);
let _ = std::fs::remove_file(&temp);
return match result {
Ok(path) => {
log::info!(
"Created orig tarball from the release download of {tag}: {}",
crate::ui::display_path(&path)
);
Ok(path)
}
Err(e) => Err(format!(
"the downloaded tarball of {url} is not a \
usable tar archive: {e}"
)
.into()),
};
}
Err(error) => {
log::warn!("Download from {url} failed: {error}");
last_error = Some(error);
}
}
}
Err(last_error.unwrap_or_else(|| format!("no download URL known for {forge:?}").into()))
}
/// Repack a user-provided tarball (a local path or an http(s) URL,
/// `.tar`/`.tar.gz`/`.tgz`/`.tar.bz2`/`.tbz2`/`.tar.xz`) into
/// `../<name>_<uver>.orig.tar.xz`.
fn fetch_and_repack(
source: &str,
name: &str,
upstream_version: &str,
tree: &Path,
) -> Result<PathBuf, Box<dyn std::error::Error>> {
let temp;
let path: &Path = if source.starts_with("http://") || source.starts_with("https://") {
log::info!("Downloading the user-provided tarball from {source}");
temp = download_to_temp(source)?;
&temp
} else {
Path::new(source)
};
let dest = repack_tarball_file(path, name, upstream_version, tree)?;
log::info!(
"Created orig tarball from {}: {}",
source,
crate::ui::display_path(&dest)
);
Ok(dest)
}
/// Download `url` into a fresh temporary file, returning its path. The
/// blocking client must not run on a tokio worker thread (scaffold is
/// called from inside the async runtime), so the download runs on a plain
/// dedicated thread.
///
/// Only the connection carries a timeout (10 s, like the rest of the
/// codebase): a whole-request timeout would cap the ENTIRE download and
/// always fail large release tarballs on slow links. The body is instead
/// streamed to the temporary file chunk by chunk as it arrives, never
/// buffered whole in memory.
fn download_to_temp(url: &str) -> Result<PathBuf, Box<dyn std::error::Error>> {
let url = url.to_string();
let download = move || -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> {
let mut response = reqwest::blocking::Client::builder()
.connect_timeout(Duration::from_secs(10))
.build()?
.get(&url)
.send()?
.error_for_status()?;
let temp = std::env::temp_dir().join(format!(
"pkh-orig-{}-{}",
std::process::id(),
chrono::Utc::now().timestamp_nanos_opt().unwrap_or_default()
));
let mut file = std::fs::File::create(&temp)?;
if let Err(error) = response.copy_to(&mut file) {
// A failed download must not leave a partial temporary behind.
let _ = std::fs::remove_file(&temp);
return Err(error.into());
}
Ok(temp)
};
let downloaded: Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> =
std::thread::spawn(download)
.join()
.map_err(|_| -> Box<dyn std::error::Error> { "the download thread panicked".into() })?;
// The thread must carry a `Send + Sync` error box; the auto traits are
// dropped on the way out.
match downloaded {
Ok(temp) => Ok(temp),
Err(error) => Err(error),
}
}
/// Container/compression format of a tarball, detected from its content
/// or, as a fallback, from its name.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum TarballFormat {
/// gzip stream (`1f 8b`), decoded natively.
Gzip,
/// xz stream (`fd 37 7a 58 5a 00`), decoded natively.
Xz,
/// bzip2 stream (`42 5a 68`, "BZh"), decompressed through the host
/// `bzip2` binary.
Bzip2,
/// Uncompressed tar (`ustar` at offset 257).
PlainTar,
}
/// Sniff the format of the tarball at `path` from its leading magic bytes
/// rather than its name: gzip (`1f 8b`), xz (`fd 37 7a 58 5a 00`), bzip2
/// (`"BZh"`) and plain tar (`ustar` at offset 257) are recognized.
/// Returns `None` when no signature matches (or the file cannot be read).
fn sniff_tarball_format(path: &Path) -> Option<TarballFormat> {
// 257 + len("ustar"): the longest signature lives at that offset.
let mut head = [0u8; 262];
let mut file = std::fs::File::open(path).ok()?;
let mut filled = 0;
while filled < head.len() {
match file.read(&mut head[filled..]) {
Ok(0) => break,
Ok(n) => filled += n,
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
Err(_) => return None,
}
}
if head.starts_with(&[0x1f, 0x8b]) {
Some(TarballFormat::Gzip)
} else if head.starts_with(&[0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00]) {
Some(TarballFormat::Xz)
} else if head.starts_with(b"BZh") {
Some(TarballFormat::Bzip2)
} else if filled == head.len() && head[257..].starts_with(b"ustar") {
Some(TarballFormat::PlainTar)
} else {
None
}
}
/// Map a tarball file NAME to its format — the fallback for local
/// user-provided files whose content carries no recognizable magic
/// signature (downloads are always sniffed first and never rejected for a
/// missing extension). Recognizes `.tar`, `.tar.gz`/`.tgz`,
/// `.tar.xz`/`.txz` and `.tar.bz2`/`.tbz2` (case-insensitively).
fn format_from_extension(path: &Path) -> Option<TarballFormat> {
let full = path
.file_name()
.and_then(|name| name.to_str())
.unwrap_or_default()
.to_ascii_lowercase();
let extension = path
.extension()
.and_then(|ext| ext.to_str())
.unwrap_or_default()
.to_ascii_lowercase();
if full.ends_with(".tar.gz") || full.ends_with(".tgz") {
Some(TarballFormat::Gzip)
} else if full.ends_with(".tar.xz") || full.ends_with(".txz") {
Some(TarballFormat::Xz)
} else if full.ends_with(".tar.bz2") || full.ends_with(".tbz2") {
Some(TarballFormat::Bzip2)
} else if extension == "tar" {
Some(TarballFormat::PlainTar)
} else {
None
}
}
/// Open `path` as a (possibly compressed) tar stream and repack it under
/// the `<name>-<uver>/` prefix into `../<name>_<uver>.orig.tar.xz`.
/// The decompression route is decided by the file's CONTENT (magic bytes),
/// not its name: downloaded streams land in an extensionless temporary
/// file and must not be rejected for a missing extension. Only local
/// user-provided files whose content carries no recognizable signature
/// fall back to the file-name extension. bzip2 content is decompressed
/// through the host `bzip2` binary (pkh carries no bzip2 codec); gz and
/// xz are decoded natively.
fn repack_tarball_file(
path: &Path,
name: &str,
upstream_version: &str,
tree: &Path,
) -> Result<PathBuf, Box<dyn std::error::Error>> {
let dest = super::debian::orig_tarball_path(tree, name, upstream_version).ok_or_else(|| {
format!(
"cannot determine the parent directory of '{}'",
tree.display()
)
})?;
if dest.exists() {
return Err(format!(
"'{}' already exists: pkh new refuses to overwrite it.",
dest.display()
)
.into());
}
// Content decides the decompression route; the name is only a fallback
// for local files with no recognizable magic (an unrecognized download
// is rejected on its content, never on a missing extension).
let format = sniff_tarball_format(path)
.or_else(|| format_from_extension(path))
.ok_or_else(|| {
format!(
"'{}' does not look like a tarball: unrecognized content \
(expected gzip, xz, bzip2 or plain tar)",
path.display()
)
})?;
log::debug!("Tarball '{}' detected as {format:?}", path.display());
let mut bzip2_child: Option<std::process::Child> = None;
let reader: Box<dyn Read> = match format {
TarballFormat::Gzip => Box::new(flate2::read::GzDecoder::new(std::fs::File::open(path)?)),
TarballFormat::Xz => Box::new(xz2::read::XzDecoder::new(std::fs::File::open(path)?)),
TarballFormat::Bzip2 => {
let mut child = Command::new("bzip2")
.arg("-dc")
.arg(path)
.stdout(Stdio::piped())
.spawn()
.map_err(|e| {
format!(
"'.tar.bz2' tarballs need the bzip2 binary on PATH to be \
repacked: {e}"
)
})?;
let stdout = child
.stdout
.take()
.ok_or_else(|| "bzip2 produced no output".to_string())?;
bzip2_child = Some(child);
Box::new(stdout)
}
TarballFormat::PlainTar => Box::new(std::fs::File::open(path)?),
};
let result = repack_tar_stream(reader, name, upstream_version, &dest);
if let Some(mut child) = bzip2_child {
if result.is_err() {
// The repack failed and nothing reads the child's stdout
// anymore: kill it (harmless if it already exited — bzip2 may
// be blocked writing into the unread pipe) and reap it,
// instead of leaking a zombie holding an open pipe.
let _ = child.kill();
let _ = child.wait();
} else {
let status = child.wait()?;
if !status.success() {
let _ = std::fs::remove_file(&dest);
return Err(format!(
"'bzip2 -dc {}' failed with status: {status}",
path.display()
)
.into());
}
}
}
if result.is_err() {
// A failed repack must not leave a half-written tarball behind.
let _ = std::fs::remove_file(&dest);
}
result
}
/// How the source tarball organizes its entries, resolved lazily from its
/// leading entries.
#[derive(Default)]
enum SourceLayout {
/// No entry has revealed the layout yet.
#[default]
Undecided,
/// A lone top-level directory entry, held back (name and header) until
/// the next entry either confirms it as the source tarball's own
/// top-level directory (classic layout) or proves the archive flat.
Probation(OsString, Box<tar::Header>),
/// Classic layout: every entry nests under one top-level directory,
/// whose component is stripped from the repacked paths.
Nested(OsString),
/// Flat layout (`tar czf up.tar.gz file1 src/ ...`): entries already
/// sit at the top level and keep their whole path under the prefix.
Flat,
}
/// Rewrite every entry of the tar `stream` under the `<name>-<uver>/`
/// top-level directory into the xz-compressed tarball at `dest`, whatever
/// the source tarball's layout. `.git` directories and tar metadata
/// leftovers are dropped, modes travel through. Classic archives nest
/// everything under one `pkg-1.0/` directory, which is stripped; FLAT
/// archives (`tar czf up.tar.gz file1 src/`) have no such directory, and
/// the old strip-first rule dropped their entries one and all, silently
/// writing an accepted-but-empty orig: flat entries now keep their whole
/// path under the new prefix.
fn repack_tar_stream(
stream: Box<dyn Read>,
name: &str,
upstream_version: &str,
dest: &Path,
) -> Result<PathBuf, Box<dyn std::error::Error>> {
let file = std::fs::File::create(dest)?;
let encoder = XzEncoder::new(file, 6);
let mut builder = tar::Builder::new(encoder);
let prefix = format!("{name}-{upstream_version}");
let mut archive = tar::Archive::new(stream);
let mut layout = SourceLayout::Undecided;
'entries: for entry in archive.entries()? {
let mut entry = entry?;
let original = entry.path()?.to_path_buf();
// GNU tar's pax metadata leftover never travels.
if original
.file_name()
.is_some_and(|name| name == "pax_global_header")
{
continue;
}
// `.git` components are dropped wherever they appear.
let components: Vec<&std::ffi::OsStr> = original
.components()
.filter(|component| component.as_os_str() != ".git")
.map(|component| component.as_os_str())
.collect();
let Some(&first) = components.first() else {
// Nothing left, e.g. the `.git` directory entry itself:
// dropped like everything that lived inside it.
continue;
};
let tail_is_empty = components.len() == 1;
let is_dir = entry.header().entry_type() == tar::EntryType::Directory;
// The path the entry keeps under the new top-level directory: the
// source's own top-level component is stripped in the classic
// nested layout, everything else keeps its whole path.
let rest: PathBuf = 'resolve: {
match std::mem::take(&mut layout) {
SourceLayout::Undecided if tail_is_empty && is_dir => {
// A lone top-level directory entry: hold it back until
// the next entry shows whether it is the source
// tarball's own top-level directory (to strip, the
// classic `pkg-1.0/` layout) or a flat archive's
// top-level directory (to keep).
layout = SourceLayout::Probation(
first.to_os_string(),
Box::new(entry.header().clone()),
);
continue 'entries;
}
SourceLayout::Undecided if tail_is_empty => {
// A top-level file: the archive is flat.
layout = SourceLayout::Flat;
break 'resolve components.iter().collect();
}
SourceLayout::Undecided => {
// Entries nested right away, with no top-level
// directory entry: the first component names the
// source's own top-level directory.
layout = SourceLayout::Nested(first.to_os_string());
break 'resolve components[1..].iter().collect();
}
SourceLayout::Probation(claimed, mut header) => {
if first == claimed.as_os_str() && (is_dir || !tail_is_empty) {
// Confirmed: the held entry is the source tarball's
// own top-level directory, skipped as always.
layout = SourceLayout::Nested(claimed);
if tail_is_empty {
continue 'entries;
}
break 'resolve components[1..].iter().collect();
}
// Refuted: a loose file or a second top-level directory
// proves the archive flat, and the held directory is a
// real one — it travels under the prefix.
let pending = format!("{prefix}/{}", claimed.to_string_lossy());
builder.append_data(&mut header, &pending, std::io::empty())?;
layout = SourceLayout::Flat;
break 'resolve components.iter().collect();
}
SourceLayout::Nested(top) => {
let under_top = first == top.as_os_str();
layout = SourceLayout::Nested(top);
if under_top && tail_is_empty {
// The source top-level directory's own entry.
continue 'entries;
}
break 'resolve if under_top {
components[1..].iter().collect()
} else {
// Sibling top-level content in a nested archive,
// kept whole (the strip-first rule dropped it).
components.iter().collect()
};
}
SourceLayout::Flat => {
layout = SourceLayout::Flat;
break 'resolve components.iter().collect();
}
}
};
let new_path = format!("{prefix}/{}", rest.to_string_lossy());
let mut header = entry.header().clone();
match header.entry_type() {
tar::EntryType::Directory => {
builder.append_data(&mut header, &new_path, std::io::empty())?;
}
tar::EntryType::Regular | tar::EntryType::Continuous => {
builder.append_data(&mut header, &new_path, &mut entry)?;
}
tar::EntryType::Symlink | tar::EntryType::Link => {
let target = entry
.link_name()?
.ok_or_else(|| format!("'{original:?}' is a link without a target"))?;
builder.append_link(&mut header, &new_path, target)?;
}
other => {
log::warn!(
"Skipping {other:?} entry '{original:?}' while repacking \
the orig tarball"
);
}
}
}
// A top-level directory held back that no other entry ever confirmed or
// refuted (the archive holds nothing else): keep it rather than
// silently dropping it.
if let SourceLayout::Probation(claimed, mut header) = layout {
let pending = format!("{prefix}/{}", claimed.to_string_lossy());
builder.append_data(&mut header, &pending, std::io::empty())?;
}
builder
.finish()
.map_err(|e| format!("failed to write '{}': {e}", dest.display()))?;
Ok(dest.to_path_buf())
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
/// List the entry names of an xz tarball.
fn tarball_names(path: &Path) -> Vec<String> {
let mut archive = tar::Archive::new(xz2::read::XzDecoder::new(
std::fs::File::open(path).unwrap(),
));
archive
.entries()
.unwrap()
.map(|entry| {
entry
.unwrap()
.path()
.unwrap()
.to_string_lossy()
.into_owned()
})
.collect()
}
/// Append the fixture entries (plus the `oldpkg-1.0/` top-level
/// directory) to any tar builder.
fn append_fixture_entries<W: std::io::Write>(
builder: &mut tar::Builder<W>,
entries: &[(&str, &str)],
) {
let mut header = tar::Header::new_gnu();
header.set_entry_type(tar::EntryType::Directory);
header.set_size(0);
header.set_mode(0o755);
header.set_cksum();
builder
.append_data(&mut header, "oldpkg-1.0", std::io::empty())
.unwrap();
for (name, contents) in entries {
let mut header = tar::Header::new_gnu();
header.set_size(contents.len() as u64);
header.set_mode(0o644);
header.set_cksum();
builder
.append_data(
&mut header,
format!("oldpkg-1.0/{name}"),
contents.as_bytes(),
)
.unwrap();
}
}
/// Build a gz tarball with the given entries (path → contents), under
/// the `oldpkg-1.0/` top-level directory.
fn write_gz_fixture(path: &Path, entries: &[(&str, &str)]) {
let file = std::fs::File::create(path).unwrap();
let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
let mut builder = tar::Builder::new(encoder);
append_fixture_entries(&mut builder, entries);
builder.into_inner().unwrap();
}
/// Build an uncompressed tar with the given entries, under the
/// `oldpkg-1.0/` top-level directory.
fn write_plain_tar_fixture(path: &Path, entries: &[(&str, &str)]) {
let mut builder = tar::Builder::new(std::fs::File::create(path).unwrap());
append_fixture_entries(&mut builder, entries);
builder.finish().unwrap();
}
/// Build an xz tarball with the given entries, under the `oldpkg-1.0/`
/// top-level directory.
fn write_xz_fixture(path: &Path, entries: &[(&str, &str)]) {
let file = std::fs::File::create(path).unwrap();
let encoder = xz2::write::XzEncoder::new(file, 1);
let mut builder = tar::Builder::new(encoder);
append_fixture_entries(&mut builder, entries);
builder.finish().unwrap();
}
/// Build a bzip2 tarball with the given entries by piping a plain tar
/// through the host `bzip2` binary (pkh carries no bzip2 codec).
/// Returns false when the binary is unavailable; callers skip the
/// bzip2-specific assertions then.
fn write_bzip2_fixture(path: &Path, entries: &[(&str, &str)]) -> bool {
if std::process::Command::new("bzip2")
.arg("--version")
.output()
.is_err()
{
return false;
}
let plain_path = path.with_extension("tar");
write_plain_tar_fixture(&plain_path, entries);
let compressed = std::fs::File::create(path).unwrap();
let ok = std::process::Command::new("bzip2")
.arg("-zc")
.arg(&plain_path)
.stdout(Stdio::from(compressed))
.status()
.map(|status| status.success())
.unwrap_or(false);
let _ = std::fs::remove_file(&plain_path);
ok
}
#[test]
fn repack_rewrites_the_top_level_prefix() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
let source = dir.path().join("old.tar.gz");
write_gz_fixture(&source, &[("src/main.rs", "hi\n"), ("README", "readme\n")]);
let dest = repack_tarball_file(&source, "mytool", "1.2.3", &tree).unwrap();
assert_eq!(dest, dir.path().join("mytool_1.2.3.orig.tar.xz"));
let names = tarball_names(&dest);
assert!(
names.iter().any(|n| n == "mytool-1.2.3/src/main.rs"),
"{names:?}"
);
assert!(
names.iter().any(|n| n == "mytool-1.2.3/README"),
"{names:?}"
);
assert!(
!names.iter().any(|n| n.starts_with("oldpkg-1.0")),
"{names:?}"
);
}
/// Append a single top-level regular-file entry to a tar builder.
fn append_flat_file<W: std::io::Write>(
builder: &mut tar::Builder<W>,
name: &str,
contents: &str,
) {
let mut header = tar::Header::new_gnu();
header.set_size(contents.len() as u64);
header.set_mode(0o644);
header.set_cksum();
builder
.append_data(&mut header, name, contents.as_bytes())
.unwrap();
}
/// Append a single top-level directory entry to a tar builder.
fn append_flat_dir<W: std::io::Write>(builder: &mut tar::Builder<W>, name: &str) {
let mut header = tar::Header::new_gnu();
header.set_entry_type(tar::EntryType::Directory);
header.set_size(0);
header.set_mode(0o755);
header.set_cksum();
builder
.append_data(&mut header, name, std::io::empty())
.unwrap();
}
/// Regression (flat tarball origin): `tar czf up.tar.gz file1 file2`
/// archives its entries at the TOP level, with no leading directory.
/// The old strip-first-component rule reduced every entry to nothing
/// and silently wrote an accepted-but-empty orig. Flat entries must
/// travel under the canonical `<name>-<uver>/` top-level directory,
/// and no entry may remain outside it.
#[test]
fn repack_puts_flat_tarball_entries_under_the_prefix() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
let source = dir.path().join("flat.tar.gz");
{
let file = std::fs::File::create(&source).unwrap();
let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
let mut builder = tar::Builder::new(encoder);
append_flat_file(&mut builder, "file1", "one\n");
append_flat_file(&mut builder, "file2", "two\n");
builder.into_inner().unwrap();
}
let dest = repack_tarball_file(&source, "mytool", "1.0.0", &tree).unwrap();
let names = tarball_names(&dest);
assert!(names.iter().any(|n| n == "mytool-1.0.0/file1"), "{names:?}");
assert!(names.iter().any(|n| n == "mytool-1.0.0/file2"), "{names:?}");
// No entry may stay loose at the top level.
assert!(
names.iter().all(|n| n.starts_with("mytool-1.0.0/")),
"{names:?}"
);
}
/// A flat archive may also carry top-level DIRECTORIES: with `src/`,
/// `file1` and `src/main.rs` all at the top level, every path keeps its
/// shape under the prefix (`src/` → `<topdir>/src/`). The directory
/// entry comes FIRST here, so the held-back-entry mechanism must
/// refute it as the source's own top-level directory for `src/` to
/// survive — a classic nested `pkg-1.0/` first entry is still skipped,
/// as the other repack tests assert.
#[test]
fn repack_keeps_flat_top_level_directories_under_the_prefix() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
let source = dir.path().join("flat-with-dir.tar.gz");
{
let file = std::fs::File::create(&source).unwrap();
let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
let mut builder = tar::Builder::new(encoder);
append_flat_dir(&mut builder, "src");
append_flat_file(&mut builder, "file1", "one\n");
append_flat_file(&mut builder, "src/main.rs", "code\n");
builder.into_inner().unwrap();
}
let dest = repack_tarball_file(&source, "mytool", "2.0.0", &tree).unwrap();
let names = tarball_names(&dest);
assert!(names.iter().any(|n| n == "mytool-2.0.0/src"), "{names:?}");
assert!(
names.iter().any(|n| n == "mytool-2.0.0/src/main.rs"),
"{names:?}"
);
assert!(names.iter().any(|n| n == "mytool-2.0.0/file1"), "{names:?}");
assert!(
names.iter().all(|n| n.starts_with("mytool-2.0.0/")),
"{names:?}"
);
}
#[test]
fn repack_drops_git_dirs_and_unsupported_entries() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
// A gz tarball carrying a .git directory next to real sources.
let source = dir.path().join("old.tar.gz");
write_gz_fixture(
&source,
&[(".git/config", "ignored"), ("src/lib.rs", "code")],
);
let dest = repack_tarball_file(&source, "mytool", "0.1.0", &tree).unwrap();
let names = tarball_names(&dest);
assert!(
names.iter().any(|n| n == "mytool-0.1.0/src/lib.rs"),
"{names:?}"
);
assert!(!names.iter().any(|n| n.contains(".git")), "{names:?}");
}
#[test]
fn repack_rejects_unknown_extensions_and_existing_dest() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
let source = dir.path().join("upstream.tar.zip");
std::fs::write(&source, b"zip").unwrap();
let err = repack_tarball_file(&source, "mytool", "0.1.0", &tree).unwrap_err();
assert!(
err.to_string().contains("does not look like a tarball"),
"{err}"
);
// An existing destination is refused before anything is unpacked.
let source = dir.path().join("upstream.tar");
std::fs::write(&source, b"").unwrap();
std::fs::write(dir.path().join("mytool_0.1.0.orig.tar.xz"), b"old").unwrap();
let err = repack_tarball_file(&source, "mytool", "0.1.0", &tree).unwrap_err();
assert!(err.to_string().contains("already exists"), "{err}");
}
#[test]
fn repack_supports_plain_and_xz_tarballs() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
// Plain .tar.
let source = dir.path().join("upstream.tar");
{
let mut builder = tar::Builder::new(std::fs::File::create(&source).unwrap());
let mut header = tar::Header::new_gnu();
header.set_size(3);
header.set_mode(0o644);
header.set_cksum();
builder
.append_data(&mut header, "oldpkg-1.0/f.txt", "abc".as_bytes())
.unwrap();
builder.finish().unwrap();
}
let dest = repack_tarball_file(&source, "mytool", "2.0", &tree).unwrap();
assert!(tarball_names(&dest).contains(&"mytool-2.0/f.txt".to_string()));
// .tar.xz.
let source = dir.path().join("upstream.tar.xz");
{
let file = std::fs::File::create(&source).unwrap();
let encoder = xz2::write::XzEncoder::new(file, 1);
let mut builder = tar::Builder::new(encoder);
let mut header = tar::Header::new_gnu();
header.set_size(3);
header.set_mode(0o644);
header.set_cksum();
builder
.append_data(&mut header, "oldpkg-1.0/g.txt", "xyz".as_bytes())
.unwrap();
builder.finish().unwrap();
}
let dest = repack_tarball_file(&source, "mytool", "2.1", &tree).unwrap();
assert!(tarball_names(&dest).contains(&"mytool-2.1/g.txt".to_string()));
}
/// Every tarball format is recognized from its real content alone:
/// gzip (`1f 8b`), xz (`fd 37 7a 58 5a 00`), bzip2 (`"BZh"`) and
/// plain tar (`ustar` at offset 257). Garbage and truncated files
/// match nothing.
#[test]
fn sniff_detects_every_magic_signature() {
let dir = tempfile::tempdir().unwrap();
let gz = dir.path().join("f.tar.gz");
write_gz_fixture(&gz, &[("a.txt", "A")]);
assert_eq!(sniff_tarball_format(&gz), Some(TarballFormat::Gzip));
let xz = dir.path().join("f.tar.xz");
write_xz_fixture(&xz, &[("a.txt", "A")]);
assert_eq!(sniff_tarball_format(&xz), Some(TarballFormat::Xz));
// Real bzip2 bytes when the host binary exists, else the raw
// "BZh" header the sniffer keys on.
let bz2 = dir.path().join("f.tar.bz2");
if write_bzip2_fixture(&bz2, &[("a.txt", "A")]) {
assert_eq!(sniff_tarball_format(&bz2), Some(TarballFormat::Bzip2));
} else {
std::fs::write(&bz2, b"BZh9\x31\x41\x59\x26\x53\x59").unwrap();
assert_eq!(sniff_tarball_format(&bz2), Some(TarballFormat::Bzip2));
}
let tar = dir.path().join("f.tar");
write_plain_tar_fixture(&tar, &[("a.txt", "A")]);
assert_eq!(sniff_tarball_format(&tar), Some(TarballFormat::PlainTar));
// Garbage (a zip header, notably) matches nothing.
let garbage = dir.path().join("garbage.bin");
std::fs::write(&garbage, b"PK\x03\x04 definitely not a tarball").unwrap();
assert_eq!(sniff_tarball_format(&garbage), None);
// Files too short to even hold a signature.
let tiny = dir.path().join("tiny.bin");
std::fs::write(&tiny, b"no").unwrap();
assert_eq!(sniff_tarball_format(&tiny), None);
// Missing files are simply unrecognized.
assert_eq!(sniff_tarball_format(&dir.path().join("missing")), None);
}
/// The extension fallback (for local user-provided files whose content
/// carries no recognizable magic) maps every supported name, including
/// the short spellings, case-insensitively.
#[test]
fn extension_fallback_mapping() {
fn p(name: &str) -> &Path {
Path::new(name)
}
assert_eq!(
format_from_extension(p("a.tar.gz")),
Some(TarballFormat::Gzip)
);
assert_eq!(format_from_extension(p("a.TGZ")), Some(TarballFormat::Gzip));
assert_eq!(
format_from_extension(p("a.tar.xz")),
Some(TarballFormat::Xz)
);
assert_eq!(format_from_extension(p("a.txz")), Some(TarballFormat::Xz));
assert_eq!(
format_from_extension(p("a.tar.bz2")),
Some(TarballFormat::Bzip2)
);
assert_eq!(
format_from_extension(p("a.tbz2")),
Some(TarballFormat::Bzip2)
);
assert_eq!(
format_from_extension(p("a.tar")),
Some(TarballFormat::PlainTar)
);
assert_eq!(format_from_extension(p("a.tar.zip")), None);
assert_eq!(format_from_extension(p("a")), None);
}
/// Regression (release-download origin): a downloaded stream lands in
/// an EXTENSIONLESS temporary file, which the extension-based check
/// always rejected ('does not look like a tarball'), silently falling
/// through to `git archive`. Valid gzip and plain-tar content must
/// round-trip through such a name.
#[test]
fn repack_sniffs_extensionless_downloaded_tarballs() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
// A gzip stream, like a codeload tar.gz download.
let source = dir.path().join("pkh-orig-1234");
write_gz_fixture(&source, &[("src/main.rs", "hi\n")]);
let dest = repack_tarball_file(&source, "mytool", "1.0.0", &tree).unwrap();
assert!(tarball_names(&dest).contains(&"mytool-1.0.0/src/main.rs".to_string()));
// A plain tar stream, also extensionless.
let source = dir.path().join("pkh-orig-5678");
write_plain_tar_fixture(&source, &[("README", "readme\n")]);
let dest = repack_tarball_file(&source, "mytool", "1.0.1", &tree).unwrap();
assert!(tarball_names(&dest).contains(&"mytool-1.0.1/README".to_string()));
}
/// The sniffed decompression route also covers xz (and bzip2, below):
/// a valid xz stream round-trips through an extensionless name.
#[test]
fn repack_sniffs_xz_tarballs() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
let source = dir.path().join("pkh-orig-xz");
write_xz_fixture(&source, &[("x.txt", "X\n")]);
let dest = repack_tarball_file(&source, "mytool", "1.1.0", &tree).unwrap();
assert!(tarball_names(&dest).contains(&"mytool-1.1.0/x.txt".to_string()));
}
/// bzip2 content is sniffed from its "BZh" header and still
/// decompressed through the host `bzip2` binary, extensionless or not.
#[test]
fn repack_sniffs_bzip2_tarballs() {
if !std::process::Command::new("bzip2")
.arg("--version")
.output()
.map(|o| o.status.success())
.unwrap_or(false)
{
return;
}
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
let source = dir.path().join("pkh-orig-bz2");
assert!(write_bzip2_fixture(&source, &[("b.txt", "B\n")]));
let dest = repack_tarball_file(&source, "mytool", "1.2.0", &tree).unwrap();
assert!(tarball_names(&dest).contains(&"mytool-1.2.0/b.txt".to_string()));
}
/// Content sniffing takes precedence over the name: a gzip stream
/// misnamed '.tar.xz' is decoded as gzip, not xz.
#[test]
fn repack_content_sniffing_beats_the_extension() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
let source = dir.path().join("misnamed.tar.xz");
write_gz_fixture(&source, &[("f.txt", "x\n")]);
let dest = repack_tarball_file(&source, "mytool", "1.3.0", &tree).unwrap();
assert!(tarball_names(&dest).contains(&"mytool-1.3.0/f.txt".to_string()));
}
/// Unsniffable content is rejected with a clear error whether or not
/// the name carries an extension — a download can only fail on
/// unrecognized CONTENT, never on a missing extension.
#[test]
fn repack_rejects_unsniffable_content() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
// Extensionless, like a downloaded temp file.
let source = dir.path().join("pkh-orig-0000");
std::fs::write(&source, b"PK\x03\x04 zip data").unwrap();
let err = repack_tarball_file(&source, "mytool", "9.9.9", &tree).unwrap_err();
assert!(
err.to_string().contains("does not look like a tarball"),
"{err}"
);
// No half-written tarball is left behind.
assert!(!dir.path().join("mytool_9.9.9.orig.tar.xz").exists());
}
#[test]
fn repack_preserves_the_exec_bit() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
let source = dir.path().join("upstream.tar");
{
let mut builder = tar::Builder::new(std::fs::File::create(&source).unwrap());
let mut header = tar::Header::new_gnu();
header.set_size(11);
header.set_mode(0o755);
header.set_cksum();
builder
.append_data(
&mut header,
"oldpkg-1.0/run.sh",
"#!/bin/sh\nx\n".as_bytes(),
)
.unwrap();
builder.finish().unwrap();
}
let dest = repack_tarball_file(&source, "mytool", "0.5.0", &tree).unwrap();
let mut archive = tar::Archive::new(xz2::read::XzDecoder::new(
std::fs::File::open(&dest).unwrap(),
));
for entry in archive.entries().unwrap() {
let entry = entry.unwrap();
if entry.path().unwrap().ends_with("run.sh") {
assert_eq!(entry.header().mode().unwrap() & 0o111, 0o111);
}
}
}
#[test]
fn vendor_component_layout_and_overwrite_refusal() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(tree.join("vendor/serde/src")).unwrap();
std::fs::write(tree.join("vendor/serde/src/lib.rs"), "crate code").unwrap();
std::fs::write(tree.join("vendor/serde/Cargo.toml"), "[package]").unwrap();
let component = create_vendor_component(&tree, "mytool", "1.0.0").unwrap();
assert_eq!(
component,
dir.path().join("mytool_1.0.0.orig-vendor.tar.xz")
);
let names = tarball_names(&component);
assert!(
names.iter().any(|n| n == "vendor/serde/src/lib.rs"),
"{names:?}"
);
// The top-level entry is the bare `vendor/` directory.
assert!(
names.iter().any(|n| n.trim_end_matches('/') == "vendor"),
"{names:?}"
);
// A second creation refuses to overwrite the stale component.
let err = create_vendor_component(&tree, "mytool", "1.0.0").unwrap_err();
assert!(err.to_string().contains("already exists"), "{err}");
}
#[test]
fn vendor_component_needs_a_non_empty_vendor_dir() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
// No vendor/ at all.
let err = create_vendor_component(&tree, "mytool", "1.0.0").unwrap_err();
assert!(err.to_string().contains("vendor"), "{err}");
// An empty vendor/ counts as nothing.
std::fs::create_dir_all(tree.join("vendor")).unwrap();
assert!(create_vendor_component(&tree, "mytool", "1.0.0").is_err());
}
/// Regression (pkh build re-vendor retry): the component file name is
/// derived from the changelog version's UPSTREAM part, never the full
/// version. The retry hook must therefore compute
/// `<name>_0.14.0.orig-vendor.tar.xz` for changelog version
/// `0.14.0-1` — the exact file `create_vendor_component` names — or a
/// stale component silently survives the recreation.
#[test]
fn component_name_uses_the_upstream_version_part() {
use crate::debian::DebianVersion;
// `0.14.0-1`: upstream part only. NOT `0.14.0-1`
// (`DebianVersion::no_epoch()`), which was the original bug.
let version = DebianVersion::parse("0.14.0-1").unwrap();
assert_eq!(component_upstream_version(&version), "0.14.0");
// Epochs are stripped the same way.
let epochy = DebianVersion::parse("2:0.14.0-1").unwrap();
assert_eq!(component_upstream_version(&epochy), "0.14.0");
// End-to-end naming: the path the retry hook looks up and the file
// `create_vendor_component` writes are one and the same.
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(tree.join("vendor/serde")).unwrap();
std::fs::write(tree.join("vendor/serde/lib.rs"), "code").unwrap();
let created =
create_vendor_component(&tree, "mytool", component_upstream_version(&version)).unwrap();
assert_eq!(
created,
vendor_component_path(&tree, "mytool", component_upstream_version(&version)).unwrap()
);
assert_eq!(
created.file_name().unwrap(),
std::ffi::OsStr::new("mytool_0.14.0.orig-vendor.tar.xz")
);
// No revision-suffixed variant may exist next to it.
assert!(
!dir.path()
.join("mytool_0.14.0-1.orig-vendor.tar.xz")
.exists()
);
}
#[test]
fn has_vendored_dir_detection() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
assert!(!has_vendored_dir(&tree));
std::fs::create_dir_all(tree.join("vendor")).unwrap();
assert!(!has_vendored_dir(&tree)); // empty
std::fs::write(tree.join("vendor/x"), "y").unwrap();
assert!(has_vendored_dir(&tree));
}
/// The snapshot origin with `vendored_rust` excludes the top-level
/// `vendor/` from the main orig (that is what the component is for).
#[test]
fn snapshot_origin_excludes_vendor_when_vendored() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(tree.join("vendor/serde/src")).unwrap();
std::fs::write(tree.join("vendor/serde/src/lib.rs"), "code").unwrap();
std::fs::write(tree.join("Cargo.toml"), "[package]").unwrap();
let created = create_orig(&tree, "mytool", "1.0.0", &OrigOrigin::Snapshot, true).unwrap();
assert_eq!(created.label, "working tree snapshot");
let names = tarball_names(&created.path);
assert!(
names.iter().any(|n| n == "mytool-1.0.0/Cargo.toml"),
"{names:?}"
);
assert!(!names.iter().any(|n| n.contains("vendor")), "{names:?}");
// Without the vendored-rust marker the directory stays in (a
// non-rust project may legitimately carry one).
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(tree.join("src/vendor")).unwrap();
std::fs::write(tree.join("src/vendor/mod.rs"), "code").unwrap();
let created = create_orig(&tree, "mytool", "1.0.0", &OrigOrigin::Snapshot, false).unwrap();
let names = tarball_names(&created.path);
assert!(
names.iter().any(|n| n == "mytool-1.0.0/src/vendor/mod.rs"),
"{names:?}"
);
}
/// git archive origin: a scripted repo with a tag produces exactly the
/// tagged content under the `<name>-<uver>/` prefix.
#[test]
fn git_archive_origin_packs_the_tag_content() {
if std::process::Command::new("git")
.arg("--version")
.output()
.is_err()
{
return;
}
let dir = tempfile::tempdir().unwrap();
let repo = dir.path().join("mytool");
std::fs::create_dir_all(&repo).unwrap();
let git = |args: &[&str]| {
let status = std::process::Command::new("git")
.args([
"-c",
"user.name=T",
"-c",
"user.email=t@example.invalid",
"-c",
"commit.gpgsign=false",
])
.args(args)
.current_dir(&repo)
.status()
.unwrap();
assert!(status.success(), "git {args:?} failed");
};
git(&["init", "-q"]);
std::fs::write(repo.join("hello.txt"), "release\n").unwrap();
git(&["add", "hello.txt"]);
git(&["commit", "-q", "-m", "release"]);
git(&["tag", "v1.2.3"]);
// A later, uncommitted-looking file exists in the worktree but must
// NOT travel into the tag archive.
std::fs::write(repo.join("uncommitted.txt"), "dirty\n").unwrap();
let created = create_orig(
&repo,
"mytool",
"1.2.3",
&OrigOrigin::GitArchive {
tag: "v1.2.3".to_string(),
},
false,
)
.unwrap();
assert_eq!(created.label, "git archive (v1.2.3)");
let names = tarball_names(&created.path);
assert!(
names.iter().any(|n| n == "mytool-1.2.3/hello.txt"),
"{names:?}"
);
assert!(
!names.iter().any(|n| n.contains("uncommitted")),
"{names:?}"
);
}
/// A failing git archive (tag missing) is reported, not silently
/// swallowed.
#[test]
fn git_archive_origin_fails_on_a_missing_tag() {
if std::process::Command::new("git")
.arg("--version")
.output()
.is_err()
{
return;
}
let dir = tempfile::tempdir().unwrap();
let repo = dir.path().join("mytool");
std::fs::create_dir_all(&repo).unwrap();
let status = std::process::Command::new("git")
.args(["init", "-q"])
.current_dir(&repo)
.status()
.unwrap();
assert!(status.success());
let err = create_orig(
&repo,
"mytool",
"1.0.0",
&OrigOrigin::GitArchive {
tag: "v9.9.9".to_string(),
},
false,
)
.unwrap_err();
assert!(err.to_string().contains("git archive"), "{err}");
// No half-written tarball is left behind.
assert!(!dir.path().join("mytool_1.0.0.orig.tar.xz").exists());
}
/// The release origin falls through to `git archive` when the download
/// fails (offline host): the label reflects the actual origin.
#[test]
fn release_origin_falls_through_to_git_archive() {
if std::process::Command::new("git")
.arg("--version")
.output()
.is_err()
{
return;
}
let dir = tempfile::tempdir().unwrap();
let repo = dir.path().join("mytool");
std::fs::create_dir_all(&repo).unwrap();
let git = |args: &[&str]| {
let status = std::process::Command::new("git")
.args([
"-c",
"user.name=T",
"-c",
"user.email=t@example.invalid",
"-c",
"commit.gpgsign=false",
])
.args(args)
.current_dir(&repo)
.status()
.unwrap();
assert!(status.success(), "git {args:?} failed");
};
git(&["init", "-q"]);
std::fs::write(repo.join("f"), "x\n").unwrap();
git(&["add", "f"]);
git(&["commit", "-q", "-m", "first"]);
git(&["tag", "v0.1.0"]);
// An unreachable URL: the download fails quickly (connection
// refused on a reserved port), the git archive takes over.
let created = create_orig(
&repo,
"mytool",
"0.1.0",
&OrigOrigin::Release {
tag: "v0.1.0".to_string(),
forge: Forge::GitHub {
owner: "pkh-nonexistent-org".into(),
repo: "pkh-nonexistent-repo".into(),
},
},
false,
)
.unwrap();
assert_eq!(created.label, "git archive (v0.1.0)");
assert!(created.path.exists());
}
/// The provided origin accepts a local gz tarball and repacks it.
#[test]
fn provided_origin_repacks_a_local_tarball() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
let source = dir.path().join("given.tar.gz");
write_gz_fixture(&source, &[("main.rs", "fn main() {}\n")]);
let created = create_orig(
&tree,
"mytool",
"3.0.0",
&OrigOrigin::Provided {
source: source.to_string_lossy().into_owned(),
},
false,
)
.unwrap();
assert!(created.label.starts_with("user tarball"));
let names = tarball_names(&created.path);
assert!(
names.iter().any(|n| n == "mytool-3.0.0/main.rs"),
"{names:?}"
);
}
/// Sanity: a tar builder accepts empty writes for directory headers
/// (the repack path relies on it).
#[test]
fn empty_write_directory_header_roundtrip() {
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("t.tar");
let mut builder = tar::Builder::new(std::fs::File::create(&dest).unwrap());
let mut header = tar::Header::new_gnu();
header.set_entry_type(tar::EntryType::Directory);
header.set_size(0);
header.set_mode(0o755);
header.set_cksum();
builder
.append_data(&mut header, "top", std::io::empty())
.unwrap();
builder.finish().unwrap();
let mut archive = tar::Archive::new(std::fs::File::open(&dest).unwrap());
let names: Vec<String> = archive
.entries()
.unwrap()
.map(|e| e.unwrap().path().unwrap().to_string_lossy().into_owned())
.collect();
assert_eq!(names, vec!["top"]);
}
/// Ensure writes into the gz fixture builder produce a readable tarball
/// (trips on header size mismatches).
#[test]
fn gz_fixture_builder_produces_readable_tarballs() {
let dir = tempfile::tempdir().unwrap();
let source = dir.path().join("fixture.tar.gz");
write_gz_fixture(&source, &[("a.txt", "A")]);
let decoder = flate2::read::GzDecoder::new(std::fs::File::open(&source).unwrap());
let mut archive = tar::Archive::new(decoder);
let mut seen = Vec::new();
for entry in archive.entries().unwrap() {
let mut entry = entry.unwrap();
let name = entry.path().unwrap().to_string_lossy().into_owned();
seen.push(name.clone());
if name.ends_with('/') || entry.header().entry_type() == tar::EntryType::Directory {
continue;
}
let mut contents = String::new();
entry.read_to_string(&mut contents).unwrap();
assert_eq!(contents, "A");
}
assert!(seen.contains(&"oldpkg-1.0/a.txt".to_string()), "{seen:?}");
}
/// Ensure writes into the xz encoder fail loudly when the stream is
/// not a tar at all (garbage input produces a readable error path).
#[test]
fn repack_garbage_fails_cleanly() {
let dir = tempfile::tempdir().unwrap();
let tree = dir.path().join("mytool");
std::fs::create_dir_all(&tree).unwrap();
let source = dir.path().join("garbage.tar.gz");
{
let mut file = std::fs::File::create(&source).unwrap();
file.write_all(b"definitely not gzip").unwrap();
}
let result = repack_tarball_file(&source, "mytool", "1.0.0", &tree);
// Either the gzip header check fails immediately or no tarball is
// left over — both are acceptable failures, a corrupt orig is not.
if let Ok(dest) = result {
assert!(
tarball_names(&dest).is_empty() || !dest.exists(),
"garbage input must not produce a usable orig"
);
}
assert!(!dir.path().join("mytool_1.0.0.orig.tar.xz").exists());
}
}