The release-tarball download capped the whole request at 30 s (large tarballs on slow links always failed and fell through to worse origins) and buffered the entire body in memory: keep a 10 s connect timeout only and stream the body to the temp file. The bzip2 -dc child of a failing repack was neither killed nor waited on (zombie + open pipe); it is now reaped on both paths. git archive no longer pipes a stderr nobody drains (a chatty git deadlocked the archive) and any failure after the destination file was created removes the empty or partial tarball.
1651 lines
65 KiB
Rust
1651 lines
65 KiB
Rust
//! Orig tarball creation for `pkh new`, one implementation per
|
|
//! [`OrigOrigin`]: working-tree snapshot, `git archive` of a release tag,
|
|
//! download of the forge release tarball, or repack of a user-provided
|
|
//! tarball — plus the dpkg upstream component tarball holding the vendored
|
|
//! Cargo dependencies (`<name>_<uver>.orig-vendor.tar.xz`).
|
|
//!
|
|
//! Whatever the origin, the tarball always lands at
|
|
//! `../<name>_<uver>.orig.tar.xz` with `<name>-<uver>/` as its single
|
|
//! top-level directory (what dpkg-source expects), and — for a vendored
|
|
//! rust package — always excludes the generated `vendor/` tree, which
|
|
//! travels in the component tarball instead and can be regenerated
|
|
//! independently of the upstream sources.
|
|
|
|
use std::ffi::OsString;
|
|
use std::io::Read;
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::{Command, Stdio};
|
|
use std::time::Duration;
|
|
|
|
use xz2::write::XzEncoder;
|
|
|
|
use super::options::OrigOrigin;
|
|
use crate::debian::DebianVersion;
|
|
use crate::new::origin::Forge;
|
|
|
|
/// The upstream version dpkg names orig/component files after: the
|
|
/// changelog version's upstream part, with the epoch and the Debian
|
|
/// revision stripped (`1:0.14.0-1` → `0.14.0`).
|
|
///
|
|
/// dpkg-source globs `../<name>_<upstream>.orig.tar.xz` and
|
|
/// `../<name>_<upstream>.orig-<component>.tar.<ext>` — never the full
|
|
/// version — so every lookup or creation of those artifacts must derive
|
|
/// the name through here. Using `DebianVersion::no_epoch()` instead yields
|
|
/// `0.14.0-1` and silently misses the real component.
|
|
pub fn component_upstream_version(changelog_version: &DebianVersion) -> &str {
|
|
&changelog_version.upstream
|
|
}
|
|
|
|
/// Result of the orig creation: where the tarball landed and how it was
|
|
/// actually produced (the release download falls through to `git archive`
|
|
/// or a snapshot on failure, so the label can differ from the plan).
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct CreatedOrig {
|
|
/// Path of the written `<name>_<uver>.orig.tar.xz`.
|
|
pub path: PathBuf,
|
|
/// Human-readable description of the actual origin.
|
|
pub label: String,
|
|
}
|
|
|
|
/// Create the orig tarball of a quilt package according to `plan`.
|
|
///
|
|
/// `vendored_rust` marks a rust package whose tree carries a generated
|
|
/// `vendor/` directory: the snapshot origin then excludes it (the other
|
|
/// origins never contain it in the first place — upstream tags predate the
|
|
/// vendoring). Failures of an explicit user choice (`--orig-from path`)
|
|
/// are fatal; a failed release download is only a warning and falls
|
|
/// through to `git archive` of the tag, then to the snapshot.
|
|
pub fn create_orig(
|
|
tree: &Path,
|
|
name: &str,
|
|
upstream_version: &str,
|
|
plan: &OrigOrigin,
|
|
vendored_rust: bool,
|
|
) -> Result<CreatedOrig, Box<dyn std::error::Error>> {
|
|
match plan {
|
|
OrigOrigin::Snapshot => {
|
|
let path = super::debian::create_orig_tarball_excluding(
|
|
tree,
|
|
name,
|
|
upstream_version,
|
|
vendored_rust,
|
|
)?;
|
|
Ok(CreatedOrig {
|
|
path,
|
|
label: OrigOrigin::Snapshot.label(),
|
|
})
|
|
}
|
|
OrigOrigin::GitArchive { tag } => {
|
|
let path = git_archive_tarball(tree, tag, name, upstream_version)?;
|
|
Ok(CreatedOrig {
|
|
path,
|
|
label: OrigOrigin::GitArchive { tag: tag.clone() }.label(),
|
|
})
|
|
}
|
|
OrigOrigin::Release { tag, forge } => {
|
|
match download_release(forge, tag, name, upstream_version, tree) {
|
|
Ok(path) => Ok(CreatedOrig {
|
|
path,
|
|
label: OrigOrigin::Release {
|
|
tag: tag.clone(),
|
|
forge: forge.clone(),
|
|
}
|
|
.label(),
|
|
}),
|
|
Err(download_error) => {
|
|
log::warn!(
|
|
"The release tarball of {tag} could not be downloaded \
|
|
({download_error:#}); falling back to `git archive` \
|
|
of the tag, then to a working-tree snapshot"
|
|
);
|
|
match git_archive_tarball(tree, tag, name, upstream_version) {
|
|
Ok(path) => Ok(CreatedOrig {
|
|
path,
|
|
label: OrigOrigin::GitArchive { tag: tag.clone() }.label(),
|
|
}),
|
|
Err(archive_error) => {
|
|
log::warn!(
|
|
"`git archive` of {tag} failed too ({archive_error:#}); \
|
|
snapshotting the working tree instead"
|
|
);
|
|
let path = super::debian::create_orig_tarball_excluding(
|
|
tree,
|
|
name,
|
|
upstream_version,
|
|
vendored_rust,
|
|
)?;
|
|
Ok(CreatedOrig {
|
|
path,
|
|
label: OrigOrigin::Snapshot.label(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
OrigOrigin::Provided { source } => {
|
|
let path = fetch_and_repack(source, name, upstream_version, tree)?;
|
|
Ok(CreatedOrig {
|
|
path,
|
|
label: OrigOrigin::Provided {
|
|
source: source.clone(),
|
|
}
|
|
.label(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Create `../<name>_<uver>.orig-vendor.tar.xz` holding the tree's
|
|
/// `vendor/` directory under a top-level `vendor/` path — the dpkg upstream
|
|
/// component dpkg-source unpacks back into the tree next to the main orig.
|
|
/// Refuses to overwrite an existing component (stale components are
|
|
/// removed by the caller, e.g. the re-vendoring retry of `pkh build`).
|
|
pub fn create_vendor_component(
|
|
tree: &Path,
|
|
name: &str,
|
|
upstream_version: &str,
|
|
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|
let vendor = tree.join("vendor");
|
|
if !is_non_empty_dir(&vendor) {
|
|
return Err(format!(
|
|
"'{}' does not exist or is empty: nothing to put into the \
|
|
orig-vendor component",
|
|
vendor.display()
|
|
)
|
|
.into());
|
|
}
|
|
let component_path = vendor_component_path(tree, name, upstream_version).ok_or_else(|| {
|
|
format!(
|
|
"cannot determine the parent directory of '{}'",
|
|
tree.display()
|
|
)
|
|
})?;
|
|
if component_path.exists() {
|
|
return Err(format!(
|
|
"'{}' already exists: pkh new refuses to overwrite it. \
|
|
Remove the stale component first.",
|
|
component_path.display()
|
|
)
|
|
.into());
|
|
}
|
|
|
|
let file = std::fs::File::create(&component_path)?;
|
|
let encoder = XzEncoder::new(file, 6);
|
|
let mut builder = tar::Builder::new(encoder);
|
|
// Everything under vendor/ travels; no exclusions (vendored crates have
|
|
// no build leftovers) and no debian/ special case below the top level.
|
|
builder.append_dir("vendor", &vendor)?;
|
|
super::debian::append_tree(&mut builder, &vendor, "vendor", 0, &[], &[])?;
|
|
builder
|
|
.finish()
|
|
.map_err(|e| format!("failed to write '{}': {}", component_path.display(), e))?;
|
|
|
|
log::info!(
|
|
"Created vendored-dependencies component {}",
|
|
crate::ui::display_path(&component_path)
|
|
);
|
|
Ok(component_path)
|
|
}
|
|
|
|
/// Path of the dpkg upstream component holding `vendor/`, next to the tree.
|
|
pub fn vendor_component_path(tree: &Path, name: &str, upstream_version: &str) -> Option<PathBuf> {
|
|
tree.parent()
|
|
.map(|parent| parent.join(format!("{name}_{upstream_version}.orig-vendor.tar.xz")))
|
|
}
|
|
|
|
/// Whether `tree` carries a generated, non-empty `vendor/` directory.
|
|
pub fn has_vendored_dir(tree: &Path) -> bool {
|
|
is_non_empty_dir(&tree.join("vendor"))
|
|
}
|
|
|
|
fn is_non_empty_dir(path: &Path) -> bool {
|
|
path.is_dir() && std::fs::read_dir(path).is_ok_and(|mut entries| entries.next().is_some())
|
|
}
|
|
|
|
/// `git archive --format=tar --prefix=<name>-<uver>/ <tag>` compressed to
|
|
/// `../<name>_<uver>.orig.tar.xz`: offline and byte-deterministic.
|
|
fn git_archive_tarball(
|
|
repo: &Path,
|
|
tag: &str,
|
|
name: &str,
|
|
upstream_version: &str,
|
|
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|
let dest = super::debian::orig_tarball_path(repo, name, upstream_version).ok_or_else(|| {
|
|
format!(
|
|
"cannot determine the parent directory of '{}'",
|
|
repo.display()
|
|
)
|
|
})?;
|
|
if dest.exists() {
|
|
return Err(format!(
|
|
"'{}' already exists: pkh new refuses to overwrite it.",
|
|
dest.display()
|
|
)
|
|
.into());
|
|
}
|
|
|
|
// Any failure after the destination file was created (spawn error,
|
|
// broken pipe, failed git) leaves behind an empty or half-written xz
|
|
// file that is not a valid tarball: it is removed before returning.
|
|
let outcome = (|| {
|
|
let file = std::fs::File::create(&dest)?;
|
|
let mut encoder = XzEncoder::new(file, 6);
|
|
// git's stderr is inherited, not piped: nothing here drains a piped
|
|
// stderr, and a chatty git filling the 64 KiB pipe buffer would
|
|
// deadlock the archive — its diagnostics belong on the terminal
|
|
// anyway, like every other child process in this module.
|
|
let mut child = Command::new("git")
|
|
.args([
|
|
"archive",
|
|
"--format=tar",
|
|
&format!("--prefix={name}-{upstream_version}/"),
|
|
tag,
|
|
])
|
|
.current_dir(repo)
|
|
.stdout(Stdio::piped())
|
|
.spawn()
|
|
.map_err(|e| format!("failed to run 'git archive': {e}"))?;
|
|
let mut stdout = child
|
|
.stdout
|
|
.take()
|
|
.ok_or_else(|| "git archive produced no output".to_string())?;
|
|
let copied = std::io::copy(&mut stdout, &mut encoder)
|
|
.map_err(|e| format!("cannot pipe git archive into '{}': {e}", dest.display()));
|
|
// The child is reaped whatever happened to the pipe, and a copy
|
|
// error only surfaces once it has been waited on. Closing our end
|
|
// first unblocks a git still writing into the pipe (SIGPIPE);
|
|
// otherwise `wait` could hang on it forever.
|
|
drop(stdout);
|
|
let status = child.wait()?;
|
|
copied?;
|
|
encoder
|
|
.finish()
|
|
.map_err(|e| format!("failed to write '{}': {e}", dest.display()))?;
|
|
Ok(status)
|
|
})();
|
|
match outcome {
|
|
Ok(status) if status.success() => {}
|
|
Ok(status) => {
|
|
let _ = std::fs::remove_file(&dest);
|
|
return Err(format!(
|
|
"'git archive --format=tar {}' failed with status: {status} \
|
|
(is HEAD exactly on the tag '{tag}'?)",
|
|
tag
|
|
)
|
|
.into());
|
|
}
|
|
Err(error) => {
|
|
let _ = std::fs::remove_file(&dest);
|
|
return Err(error);
|
|
}
|
|
}
|
|
|
|
log::info!(
|
|
"Created orig tarball from git archive of {tag}: {}",
|
|
crate::ui::display_path(&dest)
|
|
);
|
|
Ok(dest)
|
|
}
|
|
|
|
/// Download the release tarball of `tag` from `forge` (the user's choice of
|
|
/// this origin IS the network consent) and repack it to
|
|
/// `../<name>_<uver>.orig.tar.xz`. All of the forge's candidate URLs are
|
|
/// tried before failing.
|
|
fn download_release(
|
|
forge: &Forge,
|
|
tag: &str,
|
|
name: &str,
|
|
upstream_version: &str,
|
|
tree: &Path,
|
|
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|
let urls = forge.release_tarball_urls(tag);
|
|
let mut last_error: Option<Box<dyn std::error::Error>> = None;
|
|
for url in &urls {
|
|
log::info!("Downloading the upstream release tarball from {url}");
|
|
match download_to_temp(url) {
|
|
Ok(temp) => {
|
|
let result = repack_tarball_file(&temp, name, upstream_version, tree);
|
|
let _ = std::fs::remove_file(&temp);
|
|
return match result {
|
|
Ok(path) => {
|
|
log::info!(
|
|
"Created orig tarball from the release download of {tag}: {}",
|
|
crate::ui::display_path(&path)
|
|
);
|
|
Ok(path)
|
|
}
|
|
Err(e) => Err(format!(
|
|
"the downloaded tarball of {url} is not a \
|
|
usable tar archive: {e}"
|
|
)
|
|
.into()),
|
|
};
|
|
}
|
|
Err(error) => {
|
|
log::warn!("Download from {url} failed: {error}");
|
|
last_error = Some(error);
|
|
}
|
|
}
|
|
}
|
|
Err(last_error.unwrap_or_else(|| format!("no download URL known for {forge:?}").into()))
|
|
}
|
|
|
|
/// Repack a user-provided tarball (a local path or an http(s) URL,
|
|
/// `.tar`/`.tar.gz`/`.tgz`/`.tar.bz2`/`.tbz2`/`.tar.xz`) into
|
|
/// `../<name>_<uver>.orig.tar.xz`.
|
|
fn fetch_and_repack(
|
|
source: &str,
|
|
name: &str,
|
|
upstream_version: &str,
|
|
tree: &Path,
|
|
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|
let temp;
|
|
let path: &Path = if source.starts_with("http://") || source.starts_with("https://") {
|
|
log::info!("Downloading the user-provided tarball from {source}");
|
|
temp = download_to_temp(source)?;
|
|
&temp
|
|
} else {
|
|
Path::new(source)
|
|
};
|
|
let dest = repack_tarball_file(path, name, upstream_version, tree)?;
|
|
log::info!(
|
|
"Created orig tarball from {}: {}",
|
|
source,
|
|
crate::ui::display_path(&dest)
|
|
);
|
|
Ok(dest)
|
|
}
|
|
|
|
/// Download `url` into a fresh temporary file, returning its path. The
|
|
/// blocking client must not run on a tokio worker thread (scaffold is
|
|
/// called from inside the async runtime), so the download runs on a plain
|
|
/// dedicated thread.
|
|
///
|
|
/// Only the connection carries a timeout (10 s, like the rest of the
|
|
/// codebase): a whole-request timeout would cap the ENTIRE download and
|
|
/// always fail large release tarballs on slow links. The body is instead
|
|
/// streamed to the temporary file chunk by chunk as it arrives, never
|
|
/// buffered whole in memory.
|
|
fn download_to_temp(url: &str) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|
let url = url.to_string();
|
|
let download = move || -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> {
|
|
let mut response = reqwest::blocking::Client::builder()
|
|
.connect_timeout(Duration::from_secs(10))
|
|
.build()?
|
|
.get(&url)
|
|
.send()?
|
|
.error_for_status()?;
|
|
let temp = std::env::temp_dir().join(format!(
|
|
"pkh-orig-{}-{}",
|
|
std::process::id(),
|
|
chrono::Utc::now().timestamp_nanos_opt().unwrap_or_default()
|
|
));
|
|
let mut file = std::fs::File::create(&temp)?;
|
|
if let Err(error) = response.copy_to(&mut file) {
|
|
// A failed download must not leave a partial temporary behind.
|
|
let _ = std::fs::remove_file(&temp);
|
|
return Err(error.into());
|
|
}
|
|
Ok(temp)
|
|
};
|
|
let downloaded: Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> =
|
|
std::thread::spawn(download)
|
|
.join()
|
|
.map_err(|_| -> Box<dyn std::error::Error> { "the download thread panicked".into() })?;
|
|
// The thread must carry a `Send + Sync` error box; the auto traits are
|
|
// dropped on the way out.
|
|
match downloaded {
|
|
Ok(temp) => Ok(temp),
|
|
Err(error) => Err(error),
|
|
}
|
|
}
|
|
|
|
/// Container/compression format of a tarball, detected from its content
|
|
/// or, as a fallback, from its name.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
enum TarballFormat {
|
|
/// gzip stream (`1f 8b`), decoded natively.
|
|
Gzip,
|
|
/// xz stream (`fd 37 7a 58 5a 00`), decoded natively.
|
|
Xz,
|
|
/// bzip2 stream (`42 5a 68`, "BZh"), decompressed through the host
|
|
/// `bzip2` binary.
|
|
Bzip2,
|
|
/// Uncompressed tar (`ustar` at offset 257).
|
|
PlainTar,
|
|
}
|
|
|
|
/// Sniff the format of the tarball at `path` from its leading magic bytes
|
|
/// rather than its name: gzip (`1f 8b`), xz (`fd 37 7a 58 5a 00`), bzip2
|
|
/// (`"BZh"`) and plain tar (`ustar` at offset 257) are recognized.
|
|
/// Returns `None` when no signature matches (or the file cannot be read).
|
|
fn sniff_tarball_format(path: &Path) -> Option<TarballFormat> {
|
|
// 257 + len("ustar"): the longest signature lives at that offset.
|
|
let mut head = [0u8; 262];
|
|
let mut file = std::fs::File::open(path).ok()?;
|
|
let mut filled = 0;
|
|
while filled < head.len() {
|
|
match file.read(&mut head[filled..]) {
|
|
Ok(0) => break,
|
|
Ok(n) => filled += n,
|
|
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
|
|
Err(_) => return None,
|
|
}
|
|
}
|
|
if head.starts_with(&[0x1f, 0x8b]) {
|
|
Some(TarballFormat::Gzip)
|
|
} else if head.starts_with(&[0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00]) {
|
|
Some(TarballFormat::Xz)
|
|
} else if head.starts_with(b"BZh") {
|
|
Some(TarballFormat::Bzip2)
|
|
} else if filled == head.len() && head[257..].starts_with(b"ustar") {
|
|
Some(TarballFormat::PlainTar)
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
|
|
/// Map a tarball file NAME to its format — the fallback for local
|
|
/// user-provided files whose content carries no recognizable magic
|
|
/// signature (downloads are always sniffed first and never rejected for a
|
|
/// missing extension). Recognizes `.tar`, `.tar.gz`/`.tgz`,
|
|
/// `.tar.xz`/`.txz` and `.tar.bz2`/`.tbz2` (case-insensitively).
|
|
fn format_from_extension(path: &Path) -> Option<TarballFormat> {
|
|
let full = path
|
|
.file_name()
|
|
.and_then(|name| name.to_str())
|
|
.unwrap_or_default()
|
|
.to_ascii_lowercase();
|
|
let extension = path
|
|
.extension()
|
|
.and_then(|ext| ext.to_str())
|
|
.unwrap_or_default()
|
|
.to_ascii_lowercase();
|
|
if full.ends_with(".tar.gz") || full.ends_with(".tgz") {
|
|
Some(TarballFormat::Gzip)
|
|
} else if full.ends_with(".tar.xz") || full.ends_with(".txz") {
|
|
Some(TarballFormat::Xz)
|
|
} else if full.ends_with(".tar.bz2") || full.ends_with(".tbz2") {
|
|
Some(TarballFormat::Bzip2)
|
|
} else if extension == "tar" {
|
|
Some(TarballFormat::PlainTar)
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
|
|
/// Open `path` as a (possibly compressed) tar stream and repack it under
|
|
/// the `<name>-<uver>/` prefix into `../<name>_<uver>.orig.tar.xz`.
|
|
/// The decompression route is decided by the file's CONTENT (magic bytes),
|
|
/// not its name: downloaded streams land in an extensionless temporary
|
|
/// file and must not be rejected for a missing extension. Only local
|
|
/// user-provided files whose content carries no recognizable signature
|
|
/// fall back to the file-name extension. bzip2 content is decompressed
|
|
/// through the host `bzip2` binary (pkh carries no bzip2 codec); gz and
|
|
/// xz are decoded natively.
|
|
fn repack_tarball_file(
|
|
path: &Path,
|
|
name: &str,
|
|
upstream_version: &str,
|
|
tree: &Path,
|
|
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|
let dest = super::debian::orig_tarball_path(tree, name, upstream_version).ok_or_else(|| {
|
|
format!(
|
|
"cannot determine the parent directory of '{}'",
|
|
tree.display()
|
|
)
|
|
})?;
|
|
if dest.exists() {
|
|
return Err(format!(
|
|
"'{}' already exists: pkh new refuses to overwrite it.",
|
|
dest.display()
|
|
)
|
|
.into());
|
|
}
|
|
|
|
// Content decides the decompression route; the name is only a fallback
|
|
// for local files with no recognizable magic (an unrecognized download
|
|
// is rejected on its content, never on a missing extension).
|
|
let format = sniff_tarball_format(path)
|
|
.or_else(|| format_from_extension(path))
|
|
.ok_or_else(|| {
|
|
format!(
|
|
"'{}' does not look like a tarball: unrecognized content \
|
|
(expected gzip, xz, bzip2 or plain tar)",
|
|
path.display()
|
|
)
|
|
})?;
|
|
log::debug!("Tarball '{}' detected as {format:?}", path.display());
|
|
|
|
let mut bzip2_child: Option<std::process::Child> = None;
|
|
let reader: Box<dyn Read> = match format {
|
|
TarballFormat::Gzip => Box::new(flate2::read::GzDecoder::new(std::fs::File::open(path)?)),
|
|
TarballFormat::Xz => Box::new(xz2::read::XzDecoder::new(std::fs::File::open(path)?)),
|
|
TarballFormat::Bzip2 => {
|
|
let mut child = Command::new("bzip2")
|
|
.arg("-dc")
|
|
.arg(path)
|
|
.stdout(Stdio::piped())
|
|
.spawn()
|
|
.map_err(|e| {
|
|
format!(
|
|
"'.tar.bz2' tarballs need the bzip2 binary on PATH to be \
|
|
repacked: {e}"
|
|
)
|
|
})?;
|
|
let stdout = child
|
|
.stdout
|
|
.take()
|
|
.ok_or_else(|| "bzip2 produced no output".to_string())?;
|
|
bzip2_child = Some(child);
|
|
Box::new(stdout)
|
|
}
|
|
TarballFormat::PlainTar => Box::new(std::fs::File::open(path)?),
|
|
};
|
|
|
|
let result = repack_tar_stream(reader, name, upstream_version, &dest);
|
|
if let Some(mut child) = bzip2_child {
|
|
if result.is_err() {
|
|
// The repack failed and nothing reads the child's stdout
|
|
// anymore: kill it (harmless if it already exited — bzip2 may
|
|
// be blocked writing into the unread pipe) and reap it,
|
|
// instead of leaking a zombie holding an open pipe.
|
|
let _ = child.kill();
|
|
let _ = child.wait();
|
|
} else {
|
|
let status = child.wait()?;
|
|
if !status.success() {
|
|
let _ = std::fs::remove_file(&dest);
|
|
return Err(format!(
|
|
"'bzip2 -dc {}' failed with status: {status}",
|
|
path.display()
|
|
)
|
|
.into());
|
|
}
|
|
}
|
|
}
|
|
if result.is_err() {
|
|
// A failed repack must not leave a half-written tarball behind.
|
|
let _ = std::fs::remove_file(&dest);
|
|
}
|
|
result
|
|
}
|
|
|
|
/// How the source tarball organizes its entries, resolved lazily from its
|
|
/// leading entries.
|
|
#[derive(Default)]
|
|
enum SourceLayout {
|
|
/// No entry has revealed the layout yet.
|
|
#[default]
|
|
Undecided,
|
|
/// A lone top-level directory entry, held back (name and header) until
|
|
/// the next entry either confirms it as the source tarball's own
|
|
/// top-level directory (classic layout) or proves the archive flat.
|
|
Probation(OsString, Box<tar::Header>),
|
|
/// Classic layout: every entry nests under one top-level directory,
|
|
/// whose component is stripped from the repacked paths.
|
|
Nested(OsString),
|
|
/// Flat layout (`tar czf up.tar.gz file1 src/ ...`): entries already
|
|
/// sit at the top level and keep their whole path under the prefix.
|
|
Flat,
|
|
}
|
|
|
|
/// Rewrite every entry of the tar `stream` under the `<name>-<uver>/`
|
|
/// top-level directory into the xz-compressed tarball at `dest`, whatever
|
|
/// the source tarball's layout. `.git` directories and tar metadata
|
|
/// leftovers are dropped, modes travel through. Classic archives nest
|
|
/// everything under one `pkg-1.0/` directory, which is stripped; FLAT
|
|
/// archives (`tar czf up.tar.gz file1 src/`) have no such directory, and
|
|
/// the old strip-first rule dropped their entries one and all, silently
|
|
/// writing an accepted-but-empty orig: flat entries now keep their whole
|
|
/// path under the new prefix.
|
|
fn repack_tar_stream(
|
|
stream: Box<dyn Read>,
|
|
name: &str,
|
|
upstream_version: &str,
|
|
dest: &Path,
|
|
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|
let file = std::fs::File::create(dest)?;
|
|
let encoder = XzEncoder::new(file, 6);
|
|
let mut builder = tar::Builder::new(encoder);
|
|
let prefix = format!("{name}-{upstream_version}");
|
|
|
|
let mut archive = tar::Archive::new(stream);
|
|
let mut layout = SourceLayout::Undecided;
|
|
'entries: for entry in archive.entries()? {
|
|
let mut entry = entry?;
|
|
let original = entry.path()?.to_path_buf();
|
|
// GNU tar's pax metadata leftover never travels.
|
|
if original
|
|
.file_name()
|
|
.is_some_and(|name| name == "pax_global_header")
|
|
{
|
|
continue;
|
|
}
|
|
// `.git` components are dropped wherever they appear.
|
|
let components: Vec<&std::ffi::OsStr> = original
|
|
.components()
|
|
.filter(|component| component.as_os_str() != ".git")
|
|
.map(|component| component.as_os_str())
|
|
.collect();
|
|
let Some(&first) = components.first() else {
|
|
// Nothing left, e.g. the `.git` directory entry itself:
|
|
// dropped like everything that lived inside it.
|
|
continue;
|
|
};
|
|
let tail_is_empty = components.len() == 1;
|
|
let is_dir = entry.header().entry_type() == tar::EntryType::Directory;
|
|
|
|
// The path the entry keeps under the new top-level directory: the
|
|
// source's own top-level component is stripped in the classic
|
|
// nested layout, everything else keeps its whole path.
|
|
let rest: PathBuf = 'resolve: {
|
|
match std::mem::take(&mut layout) {
|
|
SourceLayout::Undecided if tail_is_empty && is_dir => {
|
|
// A lone top-level directory entry: hold it back until
|
|
// the next entry shows whether it is the source
|
|
// tarball's own top-level directory (to strip, the
|
|
// classic `pkg-1.0/` layout) or a flat archive's
|
|
// top-level directory (to keep).
|
|
layout = SourceLayout::Probation(
|
|
first.to_os_string(),
|
|
Box::new(entry.header().clone()),
|
|
);
|
|
continue 'entries;
|
|
}
|
|
SourceLayout::Undecided if tail_is_empty => {
|
|
// A top-level file: the archive is flat.
|
|
layout = SourceLayout::Flat;
|
|
break 'resolve components.iter().collect();
|
|
}
|
|
SourceLayout::Undecided => {
|
|
// Entries nested right away, with no top-level
|
|
// directory entry: the first component names the
|
|
// source's own top-level directory.
|
|
layout = SourceLayout::Nested(first.to_os_string());
|
|
break 'resolve components[1..].iter().collect();
|
|
}
|
|
SourceLayout::Probation(claimed, mut header) => {
|
|
if first == claimed.as_os_str() && (is_dir || !tail_is_empty) {
|
|
// Confirmed: the held entry is the source tarball's
|
|
// own top-level directory, skipped as always.
|
|
layout = SourceLayout::Nested(claimed);
|
|
if tail_is_empty {
|
|
continue 'entries;
|
|
}
|
|
break 'resolve components[1..].iter().collect();
|
|
}
|
|
// Refuted: a loose file or a second top-level directory
|
|
// proves the archive flat, and the held directory is a
|
|
// real one — it travels under the prefix.
|
|
let pending = format!("{prefix}/{}", claimed.to_string_lossy());
|
|
builder.append_data(&mut header, &pending, std::io::empty())?;
|
|
layout = SourceLayout::Flat;
|
|
break 'resolve components.iter().collect();
|
|
}
|
|
SourceLayout::Nested(top) => {
|
|
let under_top = first == top.as_os_str();
|
|
layout = SourceLayout::Nested(top);
|
|
if under_top && tail_is_empty {
|
|
// The source top-level directory's own entry.
|
|
continue 'entries;
|
|
}
|
|
break 'resolve if under_top {
|
|
components[1..].iter().collect()
|
|
} else {
|
|
// Sibling top-level content in a nested archive,
|
|
// kept whole (the strip-first rule dropped it).
|
|
components.iter().collect()
|
|
};
|
|
}
|
|
SourceLayout::Flat => {
|
|
layout = SourceLayout::Flat;
|
|
break 'resolve components.iter().collect();
|
|
}
|
|
}
|
|
};
|
|
let new_path = format!("{prefix}/{}", rest.to_string_lossy());
|
|
|
|
let mut header = entry.header().clone();
|
|
match header.entry_type() {
|
|
tar::EntryType::Directory => {
|
|
builder.append_data(&mut header, &new_path, std::io::empty())?;
|
|
}
|
|
tar::EntryType::Regular | tar::EntryType::Continuous => {
|
|
builder.append_data(&mut header, &new_path, &mut entry)?;
|
|
}
|
|
tar::EntryType::Symlink | tar::EntryType::Link => {
|
|
let target = entry
|
|
.link_name()?
|
|
.ok_or_else(|| format!("'{original:?}' is a link without a target"))?;
|
|
builder.append_link(&mut header, &new_path, target)?;
|
|
}
|
|
other => {
|
|
log::warn!(
|
|
"Skipping {other:?} entry '{original:?}' while repacking \
|
|
the orig tarball"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// A top-level directory held back that no other entry ever confirmed or
|
|
// refuted (the archive holds nothing else): keep it rather than
|
|
// silently dropping it.
|
|
if let SourceLayout::Probation(claimed, mut header) = layout {
|
|
let pending = format!("{prefix}/{}", claimed.to_string_lossy());
|
|
builder.append_data(&mut header, &pending, std::io::empty())?;
|
|
}
|
|
|
|
builder
|
|
.finish()
|
|
.map_err(|e| format!("failed to write '{}': {e}", dest.display()))?;
|
|
Ok(dest.to_path_buf())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::io::Write;
|
|
|
|
/// List the entry names of an xz tarball.
|
|
fn tarball_names(path: &Path) -> Vec<String> {
|
|
let mut archive = tar::Archive::new(xz2::read::XzDecoder::new(
|
|
std::fs::File::open(path).unwrap(),
|
|
));
|
|
archive
|
|
.entries()
|
|
.unwrap()
|
|
.map(|entry| {
|
|
entry
|
|
.unwrap()
|
|
.path()
|
|
.unwrap()
|
|
.to_string_lossy()
|
|
.into_owned()
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Append the fixture entries (plus the `oldpkg-1.0/` top-level
|
|
/// directory) to any tar builder.
|
|
fn append_fixture_entries<W: std::io::Write>(
|
|
builder: &mut tar::Builder<W>,
|
|
entries: &[(&str, &str)],
|
|
) {
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_entry_type(tar::EntryType::Directory);
|
|
header.set_size(0);
|
|
header.set_mode(0o755);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(&mut header, "oldpkg-1.0", std::io::empty())
|
|
.unwrap();
|
|
for (name, contents) in entries {
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_size(contents.len() as u64);
|
|
header.set_mode(0o644);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(
|
|
&mut header,
|
|
format!("oldpkg-1.0/{name}"),
|
|
contents.as_bytes(),
|
|
)
|
|
.unwrap();
|
|
}
|
|
}
|
|
|
|
/// Build a gz tarball with the given entries (path → contents), under
|
|
/// the `oldpkg-1.0/` top-level directory.
|
|
fn write_gz_fixture(path: &Path, entries: &[(&str, &str)]) {
|
|
let file = std::fs::File::create(path).unwrap();
|
|
let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
|
|
let mut builder = tar::Builder::new(encoder);
|
|
append_fixture_entries(&mut builder, entries);
|
|
builder.into_inner().unwrap();
|
|
}
|
|
|
|
/// Build an uncompressed tar with the given entries, under the
|
|
/// `oldpkg-1.0/` top-level directory.
|
|
fn write_plain_tar_fixture(path: &Path, entries: &[(&str, &str)]) {
|
|
let mut builder = tar::Builder::new(std::fs::File::create(path).unwrap());
|
|
append_fixture_entries(&mut builder, entries);
|
|
builder.finish().unwrap();
|
|
}
|
|
|
|
/// Build an xz tarball with the given entries, under the `oldpkg-1.0/`
|
|
/// top-level directory.
|
|
fn write_xz_fixture(path: &Path, entries: &[(&str, &str)]) {
|
|
let file = std::fs::File::create(path).unwrap();
|
|
let encoder = xz2::write::XzEncoder::new(file, 1);
|
|
let mut builder = tar::Builder::new(encoder);
|
|
append_fixture_entries(&mut builder, entries);
|
|
builder.finish().unwrap();
|
|
}
|
|
|
|
/// Build a bzip2 tarball with the given entries by piping a plain tar
|
|
/// through the host `bzip2` binary (pkh carries no bzip2 codec).
|
|
/// Returns false when the binary is unavailable; callers skip the
|
|
/// bzip2-specific assertions then.
|
|
fn write_bzip2_fixture(path: &Path, entries: &[(&str, &str)]) -> bool {
|
|
if std::process::Command::new("bzip2")
|
|
.arg("--version")
|
|
.output()
|
|
.is_err()
|
|
{
|
|
return false;
|
|
}
|
|
let plain_path = path.with_extension("tar");
|
|
write_plain_tar_fixture(&plain_path, entries);
|
|
let compressed = std::fs::File::create(path).unwrap();
|
|
let ok = std::process::Command::new("bzip2")
|
|
.arg("-zc")
|
|
.arg(&plain_path)
|
|
.stdout(Stdio::from(compressed))
|
|
.status()
|
|
.map(|status| status.success())
|
|
.unwrap_or(false);
|
|
let _ = std::fs::remove_file(&plain_path);
|
|
ok
|
|
}
|
|
|
|
#[test]
|
|
fn repack_rewrites_the_top_level_prefix() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("old.tar.gz");
|
|
write_gz_fixture(&source, &[("src/main.rs", "hi\n"), ("README", "readme\n")]);
|
|
|
|
let dest = repack_tarball_file(&source, "mytool", "1.2.3", &tree).unwrap();
|
|
assert_eq!(dest, dir.path().join("mytool_1.2.3.orig.tar.xz"));
|
|
let names = tarball_names(&dest);
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-1.2.3/src/main.rs"),
|
|
"{names:?}"
|
|
);
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-1.2.3/README"),
|
|
"{names:?}"
|
|
);
|
|
assert!(
|
|
!names.iter().any(|n| n.starts_with("oldpkg-1.0")),
|
|
"{names:?}"
|
|
);
|
|
}
|
|
|
|
/// Append a single top-level regular-file entry to a tar builder.
|
|
fn append_flat_file<W: std::io::Write>(
|
|
builder: &mut tar::Builder<W>,
|
|
name: &str,
|
|
contents: &str,
|
|
) {
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_size(contents.len() as u64);
|
|
header.set_mode(0o644);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(&mut header, name, contents.as_bytes())
|
|
.unwrap();
|
|
}
|
|
|
|
/// Append a single top-level directory entry to a tar builder.
|
|
fn append_flat_dir<W: std::io::Write>(builder: &mut tar::Builder<W>, name: &str) {
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_entry_type(tar::EntryType::Directory);
|
|
header.set_size(0);
|
|
header.set_mode(0o755);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(&mut header, name, std::io::empty())
|
|
.unwrap();
|
|
}
|
|
|
|
/// Regression (flat tarball origin): `tar czf up.tar.gz file1 file2`
|
|
/// archives its entries at the TOP level, with no leading directory.
|
|
/// The old strip-first-component rule reduced every entry to nothing
|
|
/// and silently wrote an accepted-but-empty orig. Flat entries must
|
|
/// travel under the canonical `<name>-<uver>/` top-level directory,
|
|
/// and no entry may remain outside it.
|
|
#[test]
|
|
fn repack_puts_flat_tarball_entries_under_the_prefix() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("flat.tar.gz");
|
|
{
|
|
let file = std::fs::File::create(&source).unwrap();
|
|
let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
|
|
let mut builder = tar::Builder::new(encoder);
|
|
append_flat_file(&mut builder, "file1", "one\n");
|
|
append_flat_file(&mut builder, "file2", "two\n");
|
|
builder.into_inner().unwrap();
|
|
}
|
|
|
|
let dest = repack_tarball_file(&source, "mytool", "1.0.0", &tree).unwrap();
|
|
let names = tarball_names(&dest);
|
|
assert!(names.iter().any(|n| n == "mytool-1.0.0/file1"), "{names:?}");
|
|
assert!(names.iter().any(|n| n == "mytool-1.0.0/file2"), "{names:?}");
|
|
// No entry may stay loose at the top level.
|
|
assert!(
|
|
names.iter().all(|n| n.starts_with("mytool-1.0.0/")),
|
|
"{names:?}"
|
|
);
|
|
}
|
|
|
|
/// A flat archive may also carry top-level DIRECTORIES: with `src/`,
|
|
/// `file1` and `src/main.rs` all at the top level, every path keeps its
|
|
/// shape under the prefix (`src/` → `<topdir>/src/`). The directory
|
|
/// entry comes FIRST here, so the held-back-entry mechanism must
|
|
/// refute it as the source's own top-level directory for `src/` to
|
|
/// survive — a classic nested `pkg-1.0/` first entry is still skipped,
|
|
/// as the other repack tests assert.
|
|
#[test]
|
|
fn repack_keeps_flat_top_level_directories_under_the_prefix() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("flat-with-dir.tar.gz");
|
|
{
|
|
let file = std::fs::File::create(&source).unwrap();
|
|
let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
|
|
let mut builder = tar::Builder::new(encoder);
|
|
append_flat_dir(&mut builder, "src");
|
|
append_flat_file(&mut builder, "file1", "one\n");
|
|
append_flat_file(&mut builder, "src/main.rs", "code\n");
|
|
builder.into_inner().unwrap();
|
|
}
|
|
|
|
let dest = repack_tarball_file(&source, "mytool", "2.0.0", &tree).unwrap();
|
|
let names = tarball_names(&dest);
|
|
assert!(names.iter().any(|n| n == "mytool-2.0.0/src"), "{names:?}");
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-2.0.0/src/main.rs"),
|
|
"{names:?}"
|
|
);
|
|
assert!(names.iter().any(|n| n == "mytool-2.0.0/file1"), "{names:?}");
|
|
assert!(
|
|
names.iter().all(|n| n.starts_with("mytool-2.0.0/")),
|
|
"{names:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn repack_drops_git_dirs_and_unsupported_entries() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
// A gz tarball carrying a .git directory next to real sources.
|
|
let source = dir.path().join("old.tar.gz");
|
|
write_gz_fixture(
|
|
&source,
|
|
&[(".git/config", "ignored"), ("src/lib.rs", "code")],
|
|
);
|
|
|
|
let dest = repack_tarball_file(&source, "mytool", "0.1.0", &tree).unwrap();
|
|
let names = tarball_names(&dest);
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-0.1.0/src/lib.rs"),
|
|
"{names:?}"
|
|
);
|
|
assert!(!names.iter().any(|n| n.contains(".git")), "{names:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn repack_rejects_unknown_extensions_and_existing_dest() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("upstream.tar.zip");
|
|
std::fs::write(&source, b"zip").unwrap();
|
|
let err = repack_tarball_file(&source, "mytool", "0.1.0", &tree).unwrap_err();
|
|
assert!(
|
|
err.to_string().contains("does not look like a tarball"),
|
|
"{err}"
|
|
);
|
|
|
|
// An existing destination is refused before anything is unpacked.
|
|
let source = dir.path().join("upstream.tar");
|
|
std::fs::write(&source, b"").unwrap();
|
|
std::fs::write(dir.path().join("mytool_0.1.0.orig.tar.xz"), b"old").unwrap();
|
|
let err = repack_tarball_file(&source, "mytool", "0.1.0", &tree).unwrap_err();
|
|
assert!(err.to_string().contains("already exists"), "{err}");
|
|
}
|
|
|
|
#[test]
|
|
fn repack_supports_plain_and_xz_tarballs() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
// Plain .tar.
|
|
let source = dir.path().join("upstream.tar");
|
|
{
|
|
let mut builder = tar::Builder::new(std::fs::File::create(&source).unwrap());
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_size(3);
|
|
header.set_mode(0o644);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(&mut header, "oldpkg-1.0/f.txt", "abc".as_bytes())
|
|
.unwrap();
|
|
builder.finish().unwrap();
|
|
}
|
|
let dest = repack_tarball_file(&source, "mytool", "2.0", &tree).unwrap();
|
|
assert!(tarball_names(&dest).contains(&"mytool-2.0/f.txt".to_string()));
|
|
|
|
// .tar.xz.
|
|
let source = dir.path().join("upstream.tar.xz");
|
|
{
|
|
let file = std::fs::File::create(&source).unwrap();
|
|
let encoder = xz2::write::XzEncoder::new(file, 1);
|
|
let mut builder = tar::Builder::new(encoder);
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_size(3);
|
|
header.set_mode(0o644);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(&mut header, "oldpkg-1.0/g.txt", "xyz".as_bytes())
|
|
.unwrap();
|
|
builder.finish().unwrap();
|
|
}
|
|
let dest = repack_tarball_file(&source, "mytool", "2.1", &tree).unwrap();
|
|
assert!(tarball_names(&dest).contains(&"mytool-2.1/g.txt".to_string()));
|
|
}
|
|
|
|
/// Every tarball format is recognized from its real content alone:
|
|
/// gzip (`1f 8b`), xz (`fd 37 7a 58 5a 00`), bzip2 (`"BZh"`) and
|
|
/// plain tar (`ustar` at offset 257). Garbage and truncated files
|
|
/// match nothing.
|
|
#[test]
|
|
fn sniff_detects_every_magic_signature() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
|
|
let gz = dir.path().join("f.tar.gz");
|
|
write_gz_fixture(&gz, &[("a.txt", "A")]);
|
|
assert_eq!(sniff_tarball_format(&gz), Some(TarballFormat::Gzip));
|
|
|
|
let xz = dir.path().join("f.tar.xz");
|
|
write_xz_fixture(&xz, &[("a.txt", "A")]);
|
|
assert_eq!(sniff_tarball_format(&xz), Some(TarballFormat::Xz));
|
|
|
|
// Real bzip2 bytes when the host binary exists, else the raw
|
|
// "BZh" header the sniffer keys on.
|
|
let bz2 = dir.path().join("f.tar.bz2");
|
|
if write_bzip2_fixture(&bz2, &[("a.txt", "A")]) {
|
|
assert_eq!(sniff_tarball_format(&bz2), Some(TarballFormat::Bzip2));
|
|
} else {
|
|
std::fs::write(&bz2, b"BZh9\x31\x41\x59\x26\x53\x59").unwrap();
|
|
assert_eq!(sniff_tarball_format(&bz2), Some(TarballFormat::Bzip2));
|
|
}
|
|
|
|
let tar = dir.path().join("f.tar");
|
|
write_plain_tar_fixture(&tar, &[("a.txt", "A")]);
|
|
assert_eq!(sniff_tarball_format(&tar), Some(TarballFormat::PlainTar));
|
|
|
|
// Garbage (a zip header, notably) matches nothing.
|
|
let garbage = dir.path().join("garbage.bin");
|
|
std::fs::write(&garbage, b"PK\x03\x04 definitely not a tarball").unwrap();
|
|
assert_eq!(sniff_tarball_format(&garbage), None);
|
|
// Files too short to even hold a signature.
|
|
let tiny = dir.path().join("tiny.bin");
|
|
std::fs::write(&tiny, b"no").unwrap();
|
|
assert_eq!(sniff_tarball_format(&tiny), None);
|
|
// Missing files are simply unrecognized.
|
|
assert_eq!(sniff_tarball_format(&dir.path().join("missing")), None);
|
|
}
|
|
|
|
/// The extension fallback (for local user-provided files whose content
|
|
/// carries no recognizable magic) maps every supported name, including
|
|
/// the short spellings, case-insensitively.
|
|
#[test]
|
|
fn extension_fallback_mapping() {
|
|
fn p(name: &str) -> &Path {
|
|
Path::new(name)
|
|
}
|
|
assert_eq!(
|
|
format_from_extension(p("a.tar.gz")),
|
|
Some(TarballFormat::Gzip)
|
|
);
|
|
assert_eq!(format_from_extension(p("a.TGZ")), Some(TarballFormat::Gzip));
|
|
assert_eq!(
|
|
format_from_extension(p("a.tar.xz")),
|
|
Some(TarballFormat::Xz)
|
|
);
|
|
assert_eq!(format_from_extension(p("a.txz")), Some(TarballFormat::Xz));
|
|
assert_eq!(
|
|
format_from_extension(p("a.tar.bz2")),
|
|
Some(TarballFormat::Bzip2)
|
|
);
|
|
assert_eq!(
|
|
format_from_extension(p("a.tbz2")),
|
|
Some(TarballFormat::Bzip2)
|
|
);
|
|
assert_eq!(
|
|
format_from_extension(p("a.tar")),
|
|
Some(TarballFormat::PlainTar)
|
|
);
|
|
assert_eq!(format_from_extension(p("a.tar.zip")), None);
|
|
assert_eq!(format_from_extension(p("a")), None);
|
|
}
|
|
|
|
/// Regression (release-download origin): a downloaded stream lands in
|
|
/// an EXTENSIONLESS temporary file, which the extension-based check
|
|
/// always rejected ('does not look like a tarball'), silently falling
|
|
/// through to `git archive`. Valid gzip and plain-tar content must
|
|
/// round-trip through such a name.
|
|
#[test]
|
|
fn repack_sniffs_extensionless_downloaded_tarballs() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
// A gzip stream, like a codeload tar.gz download.
|
|
let source = dir.path().join("pkh-orig-1234");
|
|
write_gz_fixture(&source, &[("src/main.rs", "hi\n")]);
|
|
let dest = repack_tarball_file(&source, "mytool", "1.0.0", &tree).unwrap();
|
|
assert!(tarball_names(&dest).contains(&"mytool-1.0.0/src/main.rs".to_string()));
|
|
|
|
// A plain tar stream, also extensionless.
|
|
let source = dir.path().join("pkh-orig-5678");
|
|
write_plain_tar_fixture(&source, &[("README", "readme\n")]);
|
|
let dest = repack_tarball_file(&source, "mytool", "1.0.1", &tree).unwrap();
|
|
assert!(tarball_names(&dest).contains(&"mytool-1.0.1/README".to_string()));
|
|
}
|
|
|
|
/// The sniffed decompression route also covers xz (and bzip2, below):
|
|
/// a valid xz stream round-trips through an extensionless name.
|
|
#[test]
|
|
fn repack_sniffs_xz_tarballs() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("pkh-orig-xz");
|
|
write_xz_fixture(&source, &[("x.txt", "X\n")]);
|
|
let dest = repack_tarball_file(&source, "mytool", "1.1.0", &tree).unwrap();
|
|
assert!(tarball_names(&dest).contains(&"mytool-1.1.0/x.txt".to_string()));
|
|
}
|
|
|
|
/// bzip2 content is sniffed from its "BZh" header and still
|
|
/// decompressed through the host `bzip2` binary, extensionless or not.
|
|
#[test]
|
|
fn repack_sniffs_bzip2_tarballs() {
|
|
if !std::process::Command::new("bzip2")
|
|
.arg("--version")
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
{
|
|
return;
|
|
}
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("pkh-orig-bz2");
|
|
assert!(write_bzip2_fixture(&source, &[("b.txt", "B\n")]));
|
|
let dest = repack_tarball_file(&source, "mytool", "1.2.0", &tree).unwrap();
|
|
assert!(tarball_names(&dest).contains(&"mytool-1.2.0/b.txt".to_string()));
|
|
}
|
|
|
|
/// Content sniffing takes precedence over the name: a gzip stream
|
|
/// misnamed '.tar.xz' is decoded as gzip, not xz.
|
|
#[test]
|
|
fn repack_content_sniffing_beats_the_extension() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("misnamed.tar.xz");
|
|
write_gz_fixture(&source, &[("f.txt", "x\n")]);
|
|
let dest = repack_tarball_file(&source, "mytool", "1.3.0", &tree).unwrap();
|
|
assert!(tarball_names(&dest).contains(&"mytool-1.3.0/f.txt".to_string()));
|
|
}
|
|
|
|
/// Unsniffable content is rejected with a clear error whether or not
|
|
/// the name carries an extension — a download can only fail on
|
|
/// unrecognized CONTENT, never on a missing extension.
|
|
#[test]
|
|
fn repack_rejects_unsniffable_content() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
// Extensionless, like a downloaded temp file.
|
|
let source = dir.path().join("pkh-orig-0000");
|
|
std::fs::write(&source, b"PK\x03\x04 zip data").unwrap();
|
|
let err = repack_tarball_file(&source, "mytool", "9.9.9", &tree).unwrap_err();
|
|
assert!(
|
|
err.to_string().contains("does not look like a tarball"),
|
|
"{err}"
|
|
);
|
|
// No half-written tarball is left behind.
|
|
assert!(!dir.path().join("mytool_9.9.9.orig.tar.xz").exists());
|
|
}
|
|
|
|
#[test]
|
|
fn repack_preserves_the_exec_bit() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("upstream.tar");
|
|
{
|
|
let mut builder = tar::Builder::new(std::fs::File::create(&source).unwrap());
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_size(11);
|
|
header.set_mode(0o755);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(
|
|
&mut header,
|
|
"oldpkg-1.0/run.sh",
|
|
"#!/bin/sh\nx\n".as_bytes(),
|
|
)
|
|
.unwrap();
|
|
builder.finish().unwrap();
|
|
}
|
|
let dest = repack_tarball_file(&source, "mytool", "0.5.0", &tree).unwrap();
|
|
let mut archive = tar::Archive::new(xz2::read::XzDecoder::new(
|
|
std::fs::File::open(&dest).unwrap(),
|
|
));
|
|
for entry in archive.entries().unwrap() {
|
|
let entry = entry.unwrap();
|
|
if entry.path().unwrap().ends_with("run.sh") {
|
|
assert_eq!(entry.header().mode().unwrap() & 0o111, 0o111);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn vendor_component_layout_and_overwrite_refusal() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(tree.join("vendor/serde/src")).unwrap();
|
|
std::fs::write(tree.join("vendor/serde/src/lib.rs"), "crate code").unwrap();
|
|
std::fs::write(tree.join("vendor/serde/Cargo.toml"), "[package]").unwrap();
|
|
|
|
let component = create_vendor_component(&tree, "mytool", "1.0.0").unwrap();
|
|
assert_eq!(
|
|
component,
|
|
dir.path().join("mytool_1.0.0.orig-vendor.tar.xz")
|
|
);
|
|
let names = tarball_names(&component);
|
|
assert!(
|
|
names.iter().any(|n| n == "vendor/serde/src/lib.rs"),
|
|
"{names:?}"
|
|
);
|
|
// The top-level entry is the bare `vendor/` directory.
|
|
assert!(
|
|
names.iter().any(|n| n.trim_end_matches('/') == "vendor"),
|
|
"{names:?}"
|
|
);
|
|
|
|
// A second creation refuses to overwrite the stale component.
|
|
let err = create_vendor_component(&tree, "mytool", "1.0.0").unwrap_err();
|
|
assert!(err.to_string().contains("already exists"), "{err}");
|
|
}
|
|
|
|
#[test]
|
|
fn vendor_component_needs_a_non_empty_vendor_dir() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
// No vendor/ at all.
|
|
let err = create_vendor_component(&tree, "mytool", "1.0.0").unwrap_err();
|
|
assert!(err.to_string().contains("vendor"), "{err}");
|
|
// An empty vendor/ counts as nothing.
|
|
std::fs::create_dir_all(tree.join("vendor")).unwrap();
|
|
assert!(create_vendor_component(&tree, "mytool", "1.0.0").is_err());
|
|
}
|
|
|
|
/// Regression (pkh build re-vendor retry): the component file name is
|
|
/// derived from the changelog version's UPSTREAM part, never the full
|
|
/// version. The retry hook must therefore compute
|
|
/// `<name>_0.14.0.orig-vendor.tar.xz` for changelog version
|
|
/// `0.14.0-1` — the exact file `create_vendor_component` names — or a
|
|
/// stale component silently survives the recreation.
|
|
#[test]
|
|
fn component_name_uses_the_upstream_version_part() {
|
|
use crate::debian::DebianVersion;
|
|
|
|
// `0.14.0-1`: upstream part only. NOT `0.14.0-1`
|
|
// (`DebianVersion::no_epoch()`), which was the original bug.
|
|
let version = DebianVersion::parse("0.14.0-1").unwrap();
|
|
assert_eq!(component_upstream_version(&version), "0.14.0");
|
|
// Epochs are stripped the same way.
|
|
let epochy = DebianVersion::parse("2:0.14.0-1").unwrap();
|
|
assert_eq!(component_upstream_version(&epochy), "0.14.0");
|
|
|
|
// End-to-end naming: the path the retry hook looks up and the file
|
|
// `create_vendor_component` writes are one and the same.
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(tree.join("vendor/serde")).unwrap();
|
|
std::fs::write(tree.join("vendor/serde/lib.rs"), "code").unwrap();
|
|
let created =
|
|
create_vendor_component(&tree, "mytool", component_upstream_version(&version)).unwrap();
|
|
assert_eq!(
|
|
created,
|
|
vendor_component_path(&tree, "mytool", component_upstream_version(&version)).unwrap()
|
|
);
|
|
assert_eq!(
|
|
created.file_name().unwrap(),
|
|
std::ffi::OsStr::new("mytool_0.14.0.orig-vendor.tar.xz")
|
|
);
|
|
// No revision-suffixed variant may exist next to it.
|
|
assert!(
|
|
!dir.path()
|
|
.join("mytool_0.14.0-1.orig-vendor.tar.xz")
|
|
.exists()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn has_vendored_dir_detection() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
assert!(!has_vendored_dir(&tree));
|
|
std::fs::create_dir_all(tree.join("vendor")).unwrap();
|
|
assert!(!has_vendored_dir(&tree)); // empty
|
|
std::fs::write(tree.join("vendor/x"), "y").unwrap();
|
|
assert!(has_vendored_dir(&tree));
|
|
}
|
|
|
|
/// The snapshot origin with `vendored_rust` excludes the top-level
|
|
/// `vendor/` from the main orig (that is what the component is for).
|
|
#[test]
|
|
fn snapshot_origin_excludes_vendor_when_vendored() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(tree.join("vendor/serde/src")).unwrap();
|
|
std::fs::write(tree.join("vendor/serde/src/lib.rs"), "code").unwrap();
|
|
std::fs::write(tree.join("Cargo.toml"), "[package]").unwrap();
|
|
|
|
let created = create_orig(&tree, "mytool", "1.0.0", &OrigOrigin::Snapshot, true).unwrap();
|
|
assert_eq!(created.label, "working tree snapshot");
|
|
let names = tarball_names(&created.path);
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-1.0.0/Cargo.toml"),
|
|
"{names:?}"
|
|
);
|
|
assert!(!names.iter().any(|n| n.contains("vendor")), "{names:?}");
|
|
|
|
// Without the vendored-rust marker the directory stays in (a
|
|
// non-rust project may legitimately carry one).
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(tree.join("src/vendor")).unwrap();
|
|
std::fs::write(tree.join("src/vendor/mod.rs"), "code").unwrap();
|
|
let created = create_orig(&tree, "mytool", "1.0.0", &OrigOrigin::Snapshot, false).unwrap();
|
|
let names = tarball_names(&created.path);
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-1.0.0/src/vendor/mod.rs"),
|
|
"{names:?}"
|
|
);
|
|
}
|
|
|
|
/// git archive origin: a scripted repo with a tag produces exactly the
|
|
/// tagged content under the `<name>-<uver>/` prefix.
|
|
#[test]
|
|
fn git_archive_origin_packs_the_tag_content() {
|
|
if std::process::Command::new("git")
|
|
.arg("--version")
|
|
.output()
|
|
.is_err()
|
|
{
|
|
return;
|
|
}
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let repo = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&repo).unwrap();
|
|
let git = |args: &[&str]| {
|
|
let status = std::process::Command::new("git")
|
|
.args([
|
|
"-c",
|
|
"user.name=T",
|
|
"-c",
|
|
"user.email=t@example.invalid",
|
|
"-c",
|
|
"commit.gpgsign=false",
|
|
])
|
|
.args(args)
|
|
.current_dir(&repo)
|
|
.status()
|
|
.unwrap();
|
|
assert!(status.success(), "git {args:?} failed");
|
|
};
|
|
git(&["init", "-q"]);
|
|
std::fs::write(repo.join("hello.txt"), "release\n").unwrap();
|
|
git(&["add", "hello.txt"]);
|
|
git(&["commit", "-q", "-m", "release"]);
|
|
git(&["tag", "v1.2.3"]);
|
|
// A later, uncommitted-looking file exists in the worktree but must
|
|
// NOT travel into the tag archive.
|
|
std::fs::write(repo.join("uncommitted.txt"), "dirty\n").unwrap();
|
|
|
|
let created = create_orig(
|
|
&repo,
|
|
"mytool",
|
|
"1.2.3",
|
|
&OrigOrigin::GitArchive {
|
|
tag: "v1.2.3".to_string(),
|
|
},
|
|
false,
|
|
)
|
|
.unwrap();
|
|
assert_eq!(created.label, "git archive (v1.2.3)");
|
|
let names = tarball_names(&created.path);
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-1.2.3/hello.txt"),
|
|
"{names:?}"
|
|
);
|
|
assert!(
|
|
!names.iter().any(|n| n.contains("uncommitted")),
|
|
"{names:?}"
|
|
);
|
|
}
|
|
|
|
/// A failing git archive (tag missing) is reported, not silently
|
|
/// swallowed.
|
|
#[test]
|
|
fn git_archive_origin_fails_on_a_missing_tag() {
|
|
if std::process::Command::new("git")
|
|
.arg("--version")
|
|
.output()
|
|
.is_err()
|
|
{
|
|
return;
|
|
}
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let repo = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&repo).unwrap();
|
|
let status = std::process::Command::new("git")
|
|
.args(["init", "-q"])
|
|
.current_dir(&repo)
|
|
.status()
|
|
.unwrap();
|
|
assert!(status.success());
|
|
|
|
let err = create_orig(
|
|
&repo,
|
|
"mytool",
|
|
"1.0.0",
|
|
&OrigOrigin::GitArchive {
|
|
tag: "v9.9.9".to_string(),
|
|
},
|
|
false,
|
|
)
|
|
.unwrap_err();
|
|
assert!(err.to_string().contains("git archive"), "{err}");
|
|
// No half-written tarball is left behind.
|
|
assert!(!dir.path().join("mytool_1.0.0.orig.tar.xz").exists());
|
|
}
|
|
|
|
/// The release origin falls through to `git archive` when the download
|
|
/// fails (offline host): the label reflects the actual origin.
|
|
#[test]
|
|
fn release_origin_falls_through_to_git_archive() {
|
|
if std::process::Command::new("git")
|
|
.arg("--version")
|
|
.output()
|
|
.is_err()
|
|
{
|
|
return;
|
|
}
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let repo = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&repo).unwrap();
|
|
let git = |args: &[&str]| {
|
|
let status = std::process::Command::new("git")
|
|
.args([
|
|
"-c",
|
|
"user.name=T",
|
|
"-c",
|
|
"user.email=t@example.invalid",
|
|
"-c",
|
|
"commit.gpgsign=false",
|
|
])
|
|
.args(args)
|
|
.current_dir(&repo)
|
|
.status()
|
|
.unwrap();
|
|
assert!(status.success(), "git {args:?} failed");
|
|
};
|
|
git(&["init", "-q"]);
|
|
std::fs::write(repo.join("f"), "x\n").unwrap();
|
|
git(&["add", "f"]);
|
|
git(&["commit", "-q", "-m", "first"]);
|
|
git(&["tag", "v0.1.0"]);
|
|
|
|
// An unreachable URL: the download fails quickly (connection
|
|
// refused on a reserved port), the git archive takes over.
|
|
let created = create_orig(
|
|
&repo,
|
|
"mytool",
|
|
"0.1.0",
|
|
&OrigOrigin::Release {
|
|
tag: "v0.1.0".to_string(),
|
|
forge: Forge::GitHub {
|
|
owner: "pkh-nonexistent-org".into(),
|
|
repo: "pkh-nonexistent-repo".into(),
|
|
},
|
|
},
|
|
false,
|
|
)
|
|
.unwrap();
|
|
assert_eq!(created.label, "git archive (v0.1.0)");
|
|
assert!(created.path.exists());
|
|
}
|
|
|
|
/// The provided origin accepts a local gz tarball and repacks it.
|
|
#[test]
|
|
fn provided_origin_repacks_a_local_tarball() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("given.tar.gz");
|
|
write_gz_fixture(&source, &[("main.rs", "fn main() {}\n")]);
|
|
|
|
let created = create_orig(
|
|
&tree,
|
|
"mytool",
|
|
"3.0.0",
|
|
&OrigOrigin::Provided {
|
|
source: source.to_string_lossy().into_owned(),
|
|
},
|
|
false,
|
|
)
|
|
.unwrap();
|
|
assert!(created.label.starts_with("user tarball"));
|
|
let names = tarball_names(&created.path);
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-3.0.0/main.rs"),
|
|
"{names:?}"
|
|
);
|
|
}
|
|
|
|
/// Sanity: a tar builder accepts empty writes for directory headers
|
|
/// (the repack path relies on it).
|
|
#[test]
|
|
fn empty_write_directory_header_roundtrip() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let dest = dir.path().join("t.tar");
|
|
let mut builder = tar::Builder::new(std::fs::File::create(&dest).unwrap());
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_entry_type(tar::EntryType::Directory);
|
|
header.set_size(0);
|
|
header.set_mode(0o755);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(&mut header, "top", std::io::empty())
|
|
.unwrap();
|
|
builder.finish().unwrap();
|
|
let mut archive = tar::Archive::new(std::fs::File::open(&dest).unwrap());
|
|
let names: Vec<String> = archive
|
|
.entries()
|
|
.unwrap()
|
|
.map(|e| e.unwrap().path().unwrap().to_string_lossy().into_owned())
|
|
.collect();
|
|
assert_eq!(names, vec!["top"]);
|
|
}
|
|
|
|
/// Ensure writes into the gz fixture builder produce a readable tarball
|
|
/// (trips on header size mismatches).
|
|
#[test]
|
|
fn gz_fixture_builder_produces_readable_tarballs() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let source = dir.path().join("fixture.tar.gz");
|
|
write_gz_fixture(&source, &[("a.txt", "A")]);
|
|
let decoder = flate2::read::GzDecoder::new(std::fs::File::open(&source).unwrap());
|
|
let mut archive = tar::Archive::new(decoder);
|
|
let mut seen = Vec::new();
|
|
for entry in archive.entries().unwrap() {
|
|
let mut entry = entry.unwrap();
|
|
let name = entry.path().unwrap().to_string_lossy().into_owned();
|
|
seen.push(name.clone());
|
|
if name.ends_with('/') || entry.header().entry_type() == tar::EntryType::Directory {
|
|
continue;
|
|
}
|
|
let mut contents = String::new();
|
|
entry.read_to_string(&mut contents).unwrap();
|
|
assert_eq!(contents, "A");
|
|
}
|
|
assert!(seen.contains(&"oldpkg-1.0/a.txt".to_string()), "{seen:?}");
|
|
}
|
|
|
|
/// Ensure writes into the xz encoder fail loudly when the stream is
|
|
/// not a tar at all (garbage input produces a readable error path).
|
|
#[test]
|
|
fn repack_garbage_fails_cleanly() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
let source = dir.path().join("garbage.tar.gz");
|
|
{
|
|
let mut file = std::fs::File::create(&source).unwrap();
|
|
file.write_all(b"definitely not gzip").unwrap();
|
|
}
|
|
let result = repack_tarball_file(&source, "mytool", "1.0.0", &tree);
|
|
// Either the gzip header check fails immediately or no tarball is
|
|
// left over — both are acceptable failures, a corrupt orig is not.
|
|
if let Ok(dest) = result {
|
|
assert!(
|
|
tarball_names(&dest).is_empty() || !dest.exists(),
|
|
"garbage input must not produce a usable orig"
|
|
);
|
|
}
|
|
assert!(!dir.path().join("mytool_1.0.0.orig.tar.xz").exists());
|
|
}
|
|
}
|