Prompter gains interactive(), select() and text() (with the Validator type), and confirm() now propagates cancellation as Err so flows abort instead of silently taking a default when the user hits Ctrl+C. The terminal prompter implements the full port; the port also re-exports the path display helper, which is pure presentation formatting used by events and messages rather than terminal code.
1649 lines
65 KiB
Rust
1649 lines
65 KiB
Rust
//! Orig tarball creation for `pkh new`, one implementation per
|
|
//! [`OrigOrigin`]: working-tree snapshot, `git archive` of a release tag,
|
|
//! download of the forge release tarball, or repack of a user-provided
|
|
//! tarball — plus the dpkg upstream component tarball holding the vendored
|
|
//! Cargo dependencies (`<name>_<uver>.orig-vendor.tar.xz`).
|
|
//!
|
|
//! Whatever the origin, the tarball always lands at
|
|
//! `../<name>_<uver>.orig.tar.xz` with `<name>-<uver>/` as its single
|
|
//! top-level directory (what dpkg-source expects), and — for a vendored
|
|
//! rust package — always excludes the generated `vendor/` tree, which
|
|
//! travels in the component tarball instead and can be regenerated
|
|
//! independently of the upstream sources.
|
|
|
|
use std::ffi::OsString;
|
|
use std::io::Read;
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::{Command, Stdio};
|
|
use std::time::Duration;
|
|
|
|
use xz2::write::XzEncoder;
|
|
|
|
use super::options::OrigOrigin;
|
|
use crate::debian::DebianVersion;
|
|
use crate::new::origin::Forge;
|
|
|
|
/// The upstream version dpkg names orig/component files after: the
|
|
/// changelog version's upstream part, with the epoch and the Debian
|
|
/// revision stripped (`1:0.14.0-1` → `0.14.0`).
|
|
///
|
|
/// dpkg-source globs `../<name>_<upstream>.orig.tar.xz` and
|
|
/// `../<name>_<upstream>.orig-<component>.tar.<ext>` — never the full
|
|
/// version — so every lookup or creation of those artifacts must derive
|
|
/// the name through here. Using `DebianVersion::no_epoch()` instead yields
|
|
/// `0.14.0-1` and silently misses the real component.
|
|
pub fn component_upstream_version(changelog_version: &DebianVersion) -> &str {
|
|
&changelog_version.upstream
|
|
}
|
|
|
|
/// Result of the orig creation: where the tarball landed and how it was
|
|
/// actually produced (the release download falls through to `git archive`
|
|
/// or a snapshot on failure, so the label can differ from the plan).
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct CreatedOrig {
|
|
/// Path of the written `<name>_<uver>.orig.tar.xz`.
|
|
pub path: PathBuf,
|
|
/// Human-readable description of the actual origin.
|
|
pub label: String,
|
|
}
|
|
|
|
/// Create the orig tarball of a quilt package according to `plan`.
|
|
///
|
|
/// `vendored_rust` marks a rust package whose tree carries a generated
|
|
/// `vendor/` directory: the snapshot origin then excludes it (the other
|
|
/// origins never contain it in the first place — upstream tags predate the
|
|
/// vendoring). Failures of an explicit user choice (`--orig-from path`)
|
|
/// are fatal; a failed release download is only a warning and falls
|
|
/// through to `git archive` of the tag, then to the snapshot.
|
|
pub fn create_orig(
|
|
tree: &Path,
|
|
name: &str,
|
|
upstream_version: &str,
|
|
plan: &OrigOrigin,
|
|
vendored_rust: bool,
|
|
) -> Result<CreatedOrig, Box<dyn std::error::Error>> {
|
|
match plan {
|
|
OrigOrigin::Snapshot => {
|
|
let path = super::debian::create_orig_tarball_excluding(
|
|
tree,
|
|
name,
|
|
upstream_version,
|
|
vendored_rust,
|
|
)?;
|
|
Ok(CreatedOrig {
|
|
path,
|
|
label: OrigOrigin::Snapshot.label(),
|
|
})
|
|
}
|
|
OrigOrigin::GitArchive { tag } => {
|
|
let path = git_archive_tarball(tree, tag, name, upstream_version)?;
|
|
Ok(CreatedOrig {
|
|
path,
|
|
label: OrigOrigin::GitArchive { tag: tag.clone() }.label(),
|
|
})
|
|
}
|
|
OrigOrigin::Release { tag, forge } => {
|
|
match download_release(forge, tag, name, upstream_version, tree) {
|
|
Ok(path) => Ok(CreatedOrig {
|
|
path,
|
|
label: OrigOrigin::Release {
|
|
tag: tag.clone(),
|
|
forge: forge.clone(),
|
|
}
|
|
.label(),
|
|
}),
|
|
Err(download_error) => {
|
|
log::warn!(
|
|
"The release tarball of {tag} could not be downloaded \
|
|
({download_error:#}); falling back to `git archive` \
|
|
of the tag, then to a working-tree snapshot"
|
|
);
|
|
match git_archive_tarball(tree, tag, name, upstream_version) {
|
|
Ok(path) => Ok(CreatedOrig {
|
|
path,
|
|
label: OrigOrigin::GitArchive { tag: tag.clone() }.label(),
|
|
}),
|
|
Err(archive_error) => {
|
|
log::warn!(
|
|
"`git archive` of {tag} failed too ({archive_error:#}); \
|
|
snapshotting the working tree instead"
|
|
);
|
|
let path = super::debian::create_orig_tarball_excluding(
|
|
tree,
|
|
name,
|
|
upstream_version,
|
|
vendored_rust,
|
|
)?;
|
|
Ok(CreatedOrig {
|
|
path,
|
|
label: OrigOrigin::Snapshot.label(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
OrigOrigin::Provided { source } => {
|
|
let path = fetch_and_repack(source, name, upstream_version, tree)?;
|
|
Ok(CreatedOrig {
|
|
path,
|
|
label: OrigOrigin::Provided {
|
|
source: source.clone(),
|
|
}
|
|
.label(),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Create `../<name>_<uver>.orig-vendor.tar.xz` holding the tree's
|
|
/// `vendor/` directory under a top-level `vendor/` path — the dpkg upstream
|
|
/// component dpkg-source unpacks back into the tree next to the main orig.
|
|
/// Refuses to overwrite an existing component (stale components are
|
|
/// removed by the caller, e.g. the re-vendoring retry of `pkh build`).
|
|
pub fn create_vendor_component(
|
|
tree: &Path,
|
|
name: &str,
|
|
upstream_version: &str,
|
|
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|
let vendor = tree.join("vendor");
|
|
if !is_non_empty_dir(&vendor) {
|
|
return Err(format!(
|
|
"'{}' does not exist or is empty: nothing to put into the \
|
|
orig-vendor component",
|
|
vendor.display()
|
|
)
|
|
.into());
|
|
}
|
|
let component_path = vendor_component_path(tree, name, upstream_version).ok_or_else(|| {
|
|
format!(
|
|
"cannot determine the parent directory of '{}'",
|
|
tree.display()
|
|
)
|
|
})?;
|
|
if component_path.exists() {
|
|
return Err(format!(
|
|
"'{}' already exists: pkh new refuses to overwrite it. \
|
|
Remove the stale component first.",
|
|
component_path.display()
|
|
)
|
|
.into());
|
|
}
|
|
|
|
let file = std::fs::File::create(&component_path)?;
|
|
let encoder = XzEncoder::new(file, 6);
|
|
let mut builder = tar::Builder::new(encoder);
|
|
// Everything under vendor/ travels; no exclusions (vendored crates have
|
|
// no build leftovers) and no debian/ special case below the top level.
|
|
builder.append_dir("vendor", &vendor)?;
|
|
super::debian::append_tree(&mut builder, &vendor, "vendor", 0, &[], &[])?;
|
|
builder
|
|
.finish()
|
|
.map_err(|e| format!("failed to write '{}': {}", component_path.display(), e))?;
|
|
|
|
log::info!(
|
|
"Created vendored-dependencies component {}",
|
|
crate::report::display_path(&component_path)
|
|
);
|
|
Ok(component_path)
|
|
}
|
|
|
|
/// Path of the dpkg upstream component holding `vendor/`, next to the tree.
|
|
pub fn vendor_component_path(tree: &Path, name: &str, upstream_version: &str) -> Option<PathBuf> {
|
|
tree.parent()
|
|
.map(|parent| parent.join(format!("{name}_{upstream_version}.orig-vendor.tar.xz")))
|
|
}
|
|
|
|
/// Whether `tree` carries a generated, non-empty `vendor/` directory.
|
|
pub fn has_vendored_dir(tree: &Path) -> bool {
|
|
is_non_empty_dir(&tree.join("vendor"))
|
|
}
|
|
|
|
fn is_non_empty_dir(path: &Path) -> bool {
|
|
path.is_dir() && std::fs::read_dir(path).is_ok_and(|mut entries| entries.next().is_some())
|
|
}
|
|
|
|
/// `git archive --format=tar --prefix=<name>-<uver>/ <tag>` compressed to
|
|
/// `../<name>_<uver>.orig.tar.xz`: offline and byte-deterministic.
|
|
fn git_archive_tarball(
|
|
repo: &Path,
|
|
tag: &str,
|
|
name: &str,
|
|
upstream_version: &str,
|
|
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|
let dest = super::debian::orig_tarball_path(repo, name, upstream_version).ok_or_else(|| {
|
|
format!(
|
|
"cannot determine the parent directory of '{}'",
|
|
repo.display()
|
|
)
|
|
})?;
|
|
if dest.exists() {
|
|
return Err(format!(
|
|
"'{}' already exists: pkh new refuses to overwrite it.",
|
|
dest.display()
|
|
)
|
|
.into());
|
|
}
|
|
|
|
// Any failure after the destination file was created (spawn error,
|
|
// broken pipe, failed git) leaves behind an empty or half-written xz
|
|
// file that is not a valid tarball: it is removed before returning.
|
|
let outcome = (|| {
|
|
let file = std::fs::File::create(&dest)?;
|
|
let mut encoder = XzEncoder::new(file, 6);
|
|
// git's stderr is inherited, not piped: nothing here drains a piped
|
|
// stderr, and a chatty git filling the 64 KiB pipe buffer would
|
|
// deadlock the archive — its diagnostics belong on the terminal
|
|
// anyway, like every other child process in this module.
|
|
let mut child = Command::new("git")
|
|
.args([
|
|
"archive",
|
|
"--format=tar",
|
|
&format!("--prefix={name}-{upstream_version}/"),
|
|
tag,
|
|
])
|
|
.current_dir(repo)
|
|
.stdout(Stdio::piped())
|
|
.spawn()
|
|
.map_err(|e| format!("failed to run 'git archive': {e}"))?;
|
|
let mut stdout = child
|
|
.stdout
|
|
.take()
|
|
.ok_or_else(|| "git archive produced no output".to_string())?;
|
|
let copied = std::io::copy(&mut stdout, &mut encoder)
|
|
.map_err(|e| format!("cannot pipe git archive into '{}': {e}", dest.display()));
|
|
// The child is reaped whatever happened to the pipe, and a copy
|
|
// error only surfaces once it has been waited on. Closing our end
|
|
// first unblocks a git still writing into the pipe (SIGPIPE);
|
|
// otherwise `wait` could hang on it forever.
|
|
drop(stdout);
|
|
let status = child.wait()?;
|
|
copied?;
|
|
encoder
|
|
.finish()
|
|
.map_err(|e| format!("failed to write '{}': {e}", dest.display()))?;
|
|
Ok(status)
|
|
})();
|
|
match outcome {
|
|
Ok(status) if status.success() => {}
|
|
Ok(status) => {
|
|
let _ = std::fs::remove_file(&dest);
|
|
return Err(format!(
|
|
"'git archive --format=tar {}' failed with status: {status} \
|
|
(is HEAD exactly on the tag '{tag}'?)",
|
|
tag
|
|
)
|
|
.into());
|
|
}
|
|
Err(error) => {
|
|
let _ = std::fs::remove_file(&dest);
|
|
return Err(error);
|
|
}
|
|
}
|
|
|
|
log::info!(
|
|
"Created orig tarball from git archive of {tag}: {}",
|
|
crate::report::display_path(&dest)
|
|
);
|
|
Ok(dest)
|
|
}
|
|
|
|
/// Download the release tarball of `tag` from `forge` (the user's choice of
|
|
/// this origin IS the network consent) and repack it to
|
|
/// `../<name>_<uver>.orig.tar.xz`. All of the forge's candidate URLs are
|
|
/// tried before failing.
|
|
fn download_release(
|
|
forge: &Forge,
|
|
tag: &str,
|
|
name: &str,
|
|
upstream_version: &str,
|
|
tree: &Path,
|
|
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|
let urls = forge.release_tarball_urls(tag);
|
|
let mut last_error: Option<Box<dyn std::error::Error>> = None;
|
|
for url in &urls {
|
|
log::info!("Downloading the upstream release tarball from {url}");
|
|
match download_to_temp(url) {
|
|
Ok(temp) => {
|
|
let result = repack_tarball_file(&temp, name, upstream_version, tree);
|
|
let _ = std::fs::remove_file(&temp);
|
|
return match result {
|
|
Ok(path) => {
|
|
log::info!(
|
|
"Created orig tarball from the release download of {tag}: {}",
|
|
crate::report::display_path(&path)
|
|
);
|
|
Ok(path)
|
|
}
|
|
Err(e) => Err(format!(
|
|
"the downloaded tarball of {url} is not a \
|
|
usable tar archive: {e}"
|
|
)
|
|
.into()),
|
|
};
|
|
}
|
|
Err(error) => {
|
|
log::warn!("Download from {url} failed: {error}");
|
|
last_error = Some(error);
|
|
}
|
|
}
|
|
}
|
|
Err(last_error.unwrap_or_else(|| format!("no download URL known for {forge:?}").into()))
|
|
}
|
|
|
|
/// Repack a user-provided tarball (a local path or an http(s) URL,
|
|
/// `.tar`/`.tar.gz`/`.tgz`/`.tar.bz2`/`.tbz2`/`.tar.xz`) into
|
|
/// `../<name>_<uver>.orig.tar.xz`.
|
|
fn fetch_and_repack(
|
|
source: &str,
|
|
name: &str,
|
|
upstream_version: &str,
|
|
tree: &Path,
|
|
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|
let temp;
|
|
let path: &Path = if source.starts_with("http://") || source.starts_with("https://") {
|
|
log::info!("Downloading the user-provided tarball from {source}");
|
|
temp = download_to_temp(source)?;
|
|
&temp
|
|
} else {
|
|
Path::new(source)
|
|
};
|
|
let dest = repack_tarball_file(path, name, upstream_version, tree)?;
|
|
log::info!(
|
|
"Created orig tarball from {}: {}",
|
|
source,
|
|
crate::report::display_path(&dest)
|
|
);
|
|
Ok(dest)
|
|
}
|
|
|
|
/// Download `url` into a fresh temporary file, returning its path. The
|
|
/// blocking client must not run on a tokio worker thread (scaffold is
|
|
/// called from inside the async runtime), so the download runs on a plain
|
|
/// dedicated thread.
|
|
///
|
|
/// Only the connection carries a timeout (10 s, like the rest of the
|
|
/// codebase): a whole-request timeout would cap the ENTIRE download and
|
|
/// always fail large release tarballs on slow links. The body is instead
|
|
/// streamed to the temporary file chunk by chunk as it arrives, never
|
|
/// buffered whole in memory.
|
|
fn download_to_temp(url: &str) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|
let url = url.to_string();
|
|
let download = move || -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> {
|
|
let mut response = reqwest::blocking::Client::builder()
|
|
.connect_timeout(Duration::from_secs(10))
|
|
.build()?
|
|
.get(&url)
|
|
.send()?
|
|
.error_for_status()?;
|
|
let temp = std::env::temp_dir().join(format!(
|
|
"pkh-orig-{}-{}",
|
|
std::process::id(),
|
|
chrono::Utc::now().timestamp_nanos_opt().unwrap_or_default()
|
|
));
|
|
let mut file = std::fs::File::create(&temp)?;
|
|
if let Err(error) = response.copy_to(&mut file) {
|
|
// A failed download must not leave a partial temporary behind.
|
|
let _ = std::fs::remove_file(&temp);
|
|
return Err(error.into());
|
|
}
|
|
Ok(temp)
|
|
};
|
|
let downloaded: Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> =
|
|
std::thread::spawn(download)
|
|
.join()
|
|
.map_err(|_| -> Box<dyn std::error::Error> { "the download thread panicked".into() })?;
|
|
// The thread must carry a `Send + Sync` error box; the auto traits are
|
|
// dropped on the way out.
|
|
match downloaded {
|
|
Ok(temp) => Ok(temp),
|
|
Err(error) => Err(error),
|
|
}
|
|
}
|
|
|
|
/// Container/compression format of a tarball, detected from its content
|
|
/// or, as a fallback, from its name.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
enum TarballFormat {
|
|
/// gzip stream (`1f 8b`), decoded natively.
|
|
Gzip,
|
|
/// xz stream (`fd 37 7a 58 5a 00`), decoded natively.
|
|
Xz,
|
|
/// bzip2 stream (`42 5a 68`, "BZh"), decompressed through the host
|
|
/// `bzip2` binary.
|
|
Bzip2,
|
|
/// Uncompressed tar (`ustar` at offset 257).
|
|
PlainTar,
|
|
}
|
|
|
|
/// Sniff the format of the tarball at `path` from its leading magic bytes
|
|
/// rather than its name: gzip (`1f 8b`), xz (`fd 37 7a 58 5a 00`), bzip2
|
|
/// (`"BZh"`) and plain tar (`ustar` at offset 257) are recognized.
|
|
/// Returns `None` when no signature matches (or the file cannot be read).
|
|
fn sniff_tarball_format(path: &Path) -> Option<TarballFormat> {
|
|
// 257 + len("ustar"): the longest signature lives at that offset.
|
|
let mut head = [0u8; 262];
|
|
let mut file = std::fs::File::open(path).ok()?;
|
|
let mut filled = 0;
|
|
while filled < head.len() {
|
|
match file.read(&mut head[filled..]) {
|
|
Ok(0) => break,
|
|
Ok(n) => filled += n,
|
|
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
|
|
Err(_) => return None,
|
|
}
|
|
}
|
|
if head.starts_with(&[0x1f, 0x8b]) {
|
|
Some(TarballFormat::Gzip)
|
|
} else if head.starts_with(&[0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00]) {
|
|
Some(TarballFormat::Xz)
|
|
} else if head.starts_with(b"BZh") {
|
|
Some(TarballFormat::Bzip2)
|
|
} else if filled == head.len() && head[257..].starts_with(b"ustar") {
|
|
Some(TarballFormat::PlainTar)
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
|
|
/// Map a tarball file NAME to its format — the fallback for local
|
|
/// user-provided files whose content carries no recognizable magic
|
|
/// signature (downloads are always sniffed first and never rejected for a
|
|
/// missing extension). Recognizes `.tar`, `.tar.gz`/`.tgz`,
|
|
/// `.tar.xz`/`.txz` and `.tar.bz2`/`.tbz2` (case-insensitively).
|
|
fn format_from_extension(path: &Path) -> Option<TarballFormat> {
|
|
let full = path
|
|
.file_name()
|
|
.and_then(|name| name.to_str())
|
|
.unwrap_or_default()
|
|
.to_ascii_lowercase();
|
|
let extension = path
|
|
.extension()
|
|
.and_then(|ext| ext.to_str())
|
|
.unwrap_or_default()
|
|
.to_ascii_lowercase();
|
|
if full.ends_with(".tar.gz") || full.ends_with(".tgz") {
|
|
Some(TarballFormat::Gzip)
|
|
} else if full.ends_with(".tar.xz") || full.ends_with(".txz") {
|
|
Some(TarballFormat::Xz)
|
|
} else if full.ends_with(".tar.bz2") || full.ends_with(".tbz2") {
|
|
Some(TarballFormat::Bzip2)
|
|
} else if extension == "tar" {
|
|
Some(TarballFormat::PlainTar)
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
|
|
/// Open `path` as a (possibly compressed) tar stream and repack it under
|
|
/// the `<name>-<uver>/` prefix into `../<name>_<uver>.orig.tar.xz`.
|
|
/// The decompression route is decided by the file's CONTENT (magic bytes),
|
|
/// not its name: downloaded streams land in an extensionless temporary
|
|
/// file and must not be rejected for a missing extension. Only local
|
|
/// user-provided files whose content carries no recognizable signature
|
|
/// fall back to the file-name extension. bzip2 content is decompressed
|
|
/// through the host `bzip2` binary (pkh carries no bzip2 codec); gz and
|
|
/// xz are decoded natively.
|
|
fn repack_tarball_file(
|
|
path: &Path,
|
|
name: &str,
|
|
upstream_version: &str,
|
|
tree: &Path,
|
|
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|
let dest = super::debian::orig_tarball_path(tree, name, upstream_version).ok_or_else(|| {
|
|
format!(
|
|
"cannot determine the parent directory of '{}'",
|
|
tree.display()
|
|
)
|
|
})?;
|
|
if dest.exists() {
|
|
return Err(format!(
|
|
"'{}' already exists: pkh new refuses to overwrite it.",
|
|
dest.display()
|
|
)
|
|
.into());
|
|
}
|
|
|
|
// Content decides the decompression route; the name is only a fallback
|
|
// for local files with no recognizable magic (an unrecognized download
|
|
// is rejected on its content, never on a missing extension).
|
|
let format = sniff_tarball_format(path)
|
|
.or_else(|| format_from_extension(path))
|
|
.ok_or_else(|| {
|
|
format!(
|
|
"'{}' does not look like a tarball: unrecognized content \
|
|
(expected gzip, xz, bzip2 or plain tar)",
|
|
path.display()
|
|
)
|
|
})?;
|
|
log::debug!("Tarball '{}' detected as {format:?}", path.display());
|
|
|
|
let mut bzip2_child: Option<std::process::Child> = None;
|
|
let reader: Box<dyn Read> = match format {
|
|
TarballFormat::Gzip => Box::new(flate2::read::GzDecoder::new(std::fs::File::open(path)?)),
|
|
TarballFormat::Xz => Box::new(xz2::read::XzDecoder::new(std::fs::File::open(path)?)),
|
|
TarballFormat::Bzip2 => {
|
|
let mut child = Command::new("bzip2")
|
|
.arg("-dc")
|
|
.arg(path)
|
|
.stdout(Stdio::piped())
|
|
.spawn()
|
|
.map_err(|e| {
|
|
format!(
|
|
"'.tar.bz2' tarballs need the bzip2 binary on PATH to be \
|
|
repacked: {e}"
|
|
)
|
|
})?;
|
|
let stdout = child
|
|
.stdout
|
|
.take()
|
|
.ok_or_else(|| "bzip2 produced no output".to_string())?;
|
|
bzip2_child = Some(child);
|
|
Box::new(stdout)
|
|
}
|
|
TarballFormat::PlainTar => Box::new(std::fs::File::open(path)?),
|
|
};
|
|
|
|
let result = repack_tar_stream(reader, name, upstream_version, &dest);
|
|
if let Some(mut child) = bzip2_child {
|
|
if result.is_err() {
|
|
// The repack failed and nothing reads the child's stdout
|
|
// anymore: kill it (harmless if it already exited — bzip2 may
|
|
// be blocked writing into the unread pipe) and reap it,
|
|
// instead of leaking a zombie holding an open pipe.
|
|
let _ = child.kill();
|
|
let _ = child.wait();
|
|
} else {
|
|
let status = child.wait()?;
|
|
if !status.success() {
|
|
let _ = std::fs::remove_file(&dest);
|
|
return Err(format!(
|
|
"'bzip2 -dc {}' failed with status: {status}",
|
|
path.display()
|
|
)
|
|
.into());
|
|
}
|
|
}
|
|
}
|
|
if result.is_err() {
|
|
// A failed repack must not leave a half-written tarball behind.
|
|
let _ = std::fs::remove_file(&dest);
|
|
}
|
|
result
|
|
}
|
|
|
|
/// How the source tarball organizes its entries, resolved lazily from its
|
|
/// leading entries.
|
|
#[derive(Default)]
|
|
enum SourceLayout {
|
|
/// No entry has revealed the layout yet.
|
|
#[default]
|
|
Undecided,
|
|
/// A lone top-level directory entry, held back (name and header) until
|
|
/// the next entry either confirms it as the source tarball's own
|
|
/// top-level directory (classic layout) or proves the archive flat.
|
|
Probation(OsString, Box<tar::Header>),
|
|
/// Classic layout: every entry nests under one top-level directory,
|
|
/// whose component is stripped from the repacked paths.
|
|
Nested(OsString),
|
|
/// Flat layout (`tar czf up.tar.gz file1 src/ ...`): entries already
|
|
/// sit at the top level and keep their whole path under the prefix.
|
|
Flat,
|
|
}
|
|
|
|
/// Rewrite every entry of the tar `stream` under the `<name>-<uver>/`
|
|
/// top-level directory into the xz-compressed tarball at `dest`, whatever
|
|
/// the source tarball's layout. `.git` directories and tar metadata
|
|
/// leftovers are dropped, modes travel through. Classic archives nest
|
|
/// everything under one `pkg-1.0/` directory, which is stripped; FLAT
|
|
/// archives (`tar czf up.tar.gz file1 src/`) have no such directory, and
|
|
/// the old strip-first rule dropped their entries one and all, silently
|
|
/// writing an accepted-but-empty orig: flat entries now keep their whole
|
|
/// path under the new prefix.
|
|
fn repack_tar_stream(
|
|
stream: Box<dyn Read>,
|
|
name: &str,
|
|
upstream_version: &str,
|
|
dest: &Path,
|
|
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|
let file = std::fs::File::create(dest)?;
|
|
let encoder = XzEncoder::new(file, 6);
|
|
let mut builder = tar::Builder::new(encoder);
|
|
let prefix = format!("{name}-{upstream_version}");
|
|
|
|
let mut archive = tar::Archive::new(stream);
|
|
let mut layout = SourceLayout::Undecided;
|
|
'entries: for entry in archive.entries()? {
|
|
let mut entry = entry?;
|
|
let original = entry.path()?.to_path_buf();
|
|
// GNU tar's pax metadata leftover never travels.
|
|
if original
|
|
.file_name()
|
|
.is_some_and(|name| name == "pax_global_header")
|
|
{
|
|
continue;
|
|
}
|
|
// `.git` components are dropped wherever they appear.
|
|
let components: Vec<&std::ffi::OsStr> = original
|
|
.components()
|
|
.filter(|component| component.as_os_str() != ".git")
|
|
.map(|component| component.as_os_str())
|
|
.collect();
|
|
let Some(&first) = components.first() else {
|
|
// Nothing left, e.g. the `.git` directory entry itself:
|
|
// dropped like everything that lived inside it.
|
|
continue;
|
|
};
|
|
let tail_is_empty = components.len() == 1;
|
|
let is_dir = entry.header().entry_type() == tar::EntryType::Directory;
|
|
|
|
// The path the entry keeps under the new top-level directory: the
|
|
// source's own top-level component is stripped in the classic
|
|
// nested layout, everything else keeps its whole path.
|
|
let rest: PathBuf = 'resolve: {
|
|
match std::mem::take(&mut layout) {
|
|
SourceLayout::Undecided if tail_is_empty && is_dir => {
|
|
// A lone top-level directory entry: hold it back until
|
|
// the next entry shows whether it is the source
|
|
// tarball's own top-level directory (to strip, the
|
|
// classic `pkg-1.0/` layout) or a flat archive's
|
|
// top-level directory (to keep).
|
|
layout = SourceLayout::Probation(
|
|
first.to_os_string(),
|
|
Box::new(entry.header().clone()),
|
|
);
|
|
continue 'entries;
|
|
}
|
|
SourceLayout::Undecided if tail_is_empty => {
|
|
// A top-level file: the archive is flat.
|
|
layout = SourceLayout::Flat;
|
|
break 'resolve components.iter().collect();
|
|
}
|
|
SourceLayout::Undecided => {
|
|
// Entries nested right away, with no top-level
|
|
// directory entry: the first component names the
|
|
// source's own top-level directory.
|
|
layout = SourceLayout::Nested(first.to_os_string());
|
|
break 'resolve components[1..].iter().collect();
|
|
}
|
|
SourceLayout::Probation(claimed, mut header) => {
|
|
if first == claimed.as_os_str() && (is_dir || !tail_is_empty) {
|
|
// Confirmed: the held entry is the source tarball's
|
|
// own top-level directory, skipped as always.
|
|
layout = SourceLayout::Nested(claimed);
|
|
if tail_is_empty {
|
|
continue 'entries;
|
|
}
|
|
break 'resolve components[1..].iter().collect();
|
|
}
|
|
// Refuted: a loose file or a second top-level directory
|
|
// proves the archive flat, and the held directory is a
|
|
// real one — it travels under the prefix.
|
|
let pending = format!("{prefix}/{}", claimed.to_string_lossy());
|
|
builder.append_data(&mut header, &pending, std::io::empty())?;
|
|
layout = SourceLayout::Flat;
|
|
break 'resolve components.iter().collect();
|
|
}
|
|
SourceLayout::Nested(top) => {
|
|
let under_top = first == top.as_os_str();
|
|
layout = SourceLayout::Nested(top);
|
|
if under_top && tail_is_empty {
|
|
// The source top-level directory's own entry.
|
|
continue 'entries;
|
|
}
|
|
break 'resolve if under_top {
|
|
components[1..].iter().collect()
|
|
} else {
|
|
// Sibling top-level content in a nested archive,
|
|
// kept whole (the strip-first rule dropped it).
|
|
components.iter().collect()
|
|
};
|
|
}
|
|
SourceLayout::Flat => {
|
|
layout = SourceLayout::Flat;
|
|
break 'resolve components.iter().collect();
|
|
}
|
|
}
|
|
};
|
|
let new_path = format!("{prefix}/{}", rest.to_string_lossy());
|
|
|
|
let mut header = entry.header().clone();
|
|
match header.entry_type() {
|
|
tar::EntryType::Directory => {
|
|
builder.append_data(&mut header, &new_path, std::io::empty())?;
|
|
}
|
|
tar::EntryType::Regular | tar::EntryType::Continuous => {
|
|
builder.append_data(&mut header, &new_path, &mut entry)?;
|
|
}
|
|
tar::EntryType::Symlink | tar::EntryType::Link => {
|
|
let target = entry
|
|
.link_name()?
|
|
.ok_or_else(|| format!("'{original:?}' is a link without a target"))?;
|
|
builder.append_link(&mut header, &new_path, target)?;
|
|
}
|
|
other => {
|
|
log::warn!(
|
|
"Skipping {other:?} entry '{original:?}' while repacking \
|
|
the orig tarball"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// A top-level directory held back that no other entry ever confirmed or
|
|
// refuted (the archive holds nothing else): keep it rather than
|
|
// silently dropping it.
|
|
if let SourceLayout::Probation(claimed, mut header) = layout {
|
|
let pending = format!("{prefix}/{}", claimed.to_string_lossy());
|
|
builder.append_data(&mut header, &pending, std::io::empty())?;
|
|
}
|
|
|
|
builder
|
|
.finish()
|
|
.map_err(|e| format!("failed to write '{}': {e}", dest.display()))?;
|
|
Ok(dest.to_path_buf())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::io::Write;
|
|
|
|
/// List the entry names of an xz tarball.
|
|
fn tarball_names(path: &Path) -> Vec<String> {
|
|
let mut archive = tar::Archive::new(xz2::read::XzDecoder::new(
|
|
std::fs::File::open(path).unwrap(),
|
|
));
|
|
archive
|
|
.entries()
|
|
.unwrap()
|
|
.map(|entry| {
|
|
entry
|
|
.unwrap()
|
|
.path()
|
|
.unwrap()
|
|
.to_string_lossy()
|
|
.into_owned()
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Append the fixture entries (plus the `oldpkg-1.0/` top-level
|
|
/// directory) to any tar builder.
|
|
fn append_fixture_entries<W: std::io::Write>(
|
|
builder: &mut tar::Builder<W>,
|
|
entries: &[(&str, &str)],
|
|
) {
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_entry_type(tar::EntryType::Directory);
|
|
header.set_size(0);
|
|
header.set_mode(0o755);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(&mut header, "oldpkg-1.0", std::io::empty())
|
|
.unwrap();
|
|
for (name, contents) in entries {
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_size(contents.len() as u64);
|
|
header.set_mode(0o644);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(
|
|
&mut header,
|
|
format!("oldpkg-1.0/{name}"),
|
|
contents.as_bytes(),
|
|
)
|
|
.unwrap();
|
|
}
|
|
}
|
|
|
|
/// Build a gz tarball with the given entries (path → contents), under
|
|
/// the `oldpkg-1.0/` top-level directory.
|
|
fn write_gz_fixture(path: &Path, entries: &[(&str, &str)]) {
|
|
let file = std::fs::File::create(path).unwrap();
|
|
let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
|
|
let mut builder = tar::Builder::new(encoder);
|
|
append_fixture_entries(&mut builder, entries);
|
|
builder.into_inner().unwrap();
|
|
}
|
|
|
|
/// Build an uncompressed tar with the given entries, under the
|
|
/// `oldpkg-1.0/` top-level directory.
|
|
fn write_plain_tar_fixture(path: &Path, entries: &[(&str, &str)]) {
|
|
let mut builder = tar::Builder::new(std::fs::File::create(path).unwrap());
|
|
append_fixture_entries(&mut builder, entries);
|
|
builder.finish().unwrap();
|
|
}
|
|
|
|
/// Build an xz tarball with the given entries, under the `oldpkg-1.0/`
|
|
/// top-level directory.
|
|
fn write_xz_fixture(path: &Path, entries: &[(&str, &str)]) {
|
|
let file = std::fs::File::create(path).unwrap();
|
|
let encoder = xz2::write::XzEncoder::new(file, 1);
|
|
let mut builder = tar::Builder::new(encoder);
|
|
append_fixture_entries(&mut builder, entries);
|
|
builder.finish().unwrap();
|
|
}
|
|
|
|
/// Build a bzip2 tarball with the given entries by piping a plain tar
|
|
/// through the host `bzip2` binary (pkh carries no bzip2 codec).
|
|
/// Returns false when the binary is unavailable; callers skip the
|
|
/// bzip2-specific assertions then.
|
|
fn write_bzip2_fixture(path: &Path, entries: &[(&str, &str)]) -> bool {
|
|
if std::process::Command::new("bzip2")
|
|
.arg("--version")
|
|
.output()
|
|
.is_err()
|
|
{
|
|
return false;
|
|
}
|
|
let plain_path = path.with_extension("tar");
|
|
write_plain_tar_fixture(&plain_path, entries);
|
|
let compressed = std::fs::File::create(path).unwrap();
|
|
let ok = std::process::Command::new("bzip2")
|
|
.arg("-zc")
|
|
.arg(&plain_path)
|
|
.stdout(Stdio::from(compressed))
|
|
.status()
|
|
.map(|status| status.success())
|
|
.unwrap_or(false);
|
|
let _ = std::fs::remove_file(&plain_path);
|
|
ok
|
|
}
|
|
|
|
#[test]
|
|
fn repack_rewrites_the_top_level_prefix() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("old.tar.gz");
|
|
write_gz_fixture(&source, &[("src/main.rs", "hi\n"), ("README", "readme\n")]);
|
|
|
|
let dest = repack_tarball_file(&source, "mytool", "1.2.3", &tree).unwrap();
|
|
assert_eq!(dest, dir.path().join("mytool_1.2.3.orig.tar.xz"));
|
|
let names = tarball_names(&dest);
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-1.2.3/src/main.rs"),
|
|
"{names:?}"
|
|
);
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-1.2.3/README"),
|
|
"{names:?}"
|
|
);
|
|
assert!(
|
|
!names.iter().any(|n| n.starts_with("oldpkg-1.0")),
|
|
"{names:?}"
|
|
);
|
|
}
|
|
|
|
/// Append a single top-level regular-file entry to a tar builder.
|
|
fn append_flat_file<W: std::io::Write>(
|
|
builder: &mut tar::Builder<W>,
|
|
name: &str,
|
|
contents: &str,
|
|
) {
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_size(contents.len() as u64);
|
|
header.set_mode(0o644);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(&mut header, name, contents.as_bytes())
|
|
.unwrap();
|
|
}
|
|
|
|
/// Append a single top-level directory entry to a tar builder.
|
|
fn append_flat_dir<W: std::io::Write>(builder: &mut tar::Builder<W>, name: &str) {
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_entry_type(tar::EntryType::Directory);
|
|
header.set_size(0);
|
|
header.set_mode(0o755);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(&mut header, name, std::io::empty())
|
|
.unwrap();
|
|
}
|
|
|
|
/// Regression (flat tarball origin): `tar czf up.tar.gz file1 file2`
|
|
/// archives its entries at the TOP level, with no leading directory.
|
|
/// The old strip-first-component rule reduced every entry to nothing
|
|
/// and silently wrote an accepted-but-empty orig. Flat entries must
|
|
/// travel under the canonical `<name>-<uver>/` top-level directory,
|
|
/// and no entry may remain outside it.
|
|
#[test]
|
|
fn repack_puts_flat_tarball_entries_under_the_prefix() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("flat.tar.gz");
|
|
{
|
|
let file = std::fs::File::create(&source).unwrap();
|
|
let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
|
|
let mut builder = tar::Builder::new(encoder);
|
|
append_flat_file(&mut builder, "file1", "one\n");
|
|
append_flat_file(&mut builder, "file2", "two\n");
|
|
builder.into_inner().unwrap();
|
|
}
|
|
|
|
let dest = repack_tarball_file(&source, "mytool", "1.0.0", &tree).unwrap();
|
|
let names = tarball_names(&dest);
|
|
assert!(names.iter().any(|n| n == "mytool-1.0.0/file1"), "{names:?}");
|
|
assert!(names.iter().any(|n| n == "mytool-1.0.0/file2"), "{names:?}");
|
|
// No entry may stay loose at the top level.
|
|
assert!(
|
|
names.iter().all(|n| n.starts_with("mytool-1.0.0/")),
|
|
"{names:?}"
|
|
);
|
|
}
|
|
|
|
/// A flat archive may also carry top-level DIRECTORIES: with `src/`,
|
|
/// `file1` and `src/main.rs` all at the top level, every path keeps its
|
|
/// shape under the prefix (`src/` → `<topdir>/src/`). The directory
|
|
/// entry comes FIRST here, so the held-back-entry mechanism must
|
|
/// refute it as the source's own top-level directory for `src/` to
|
|
/// survive — a classic nested `pkg-1.0/` first entry is still skipped,
|
|
/// as the other repack tests assert.
|
|
#[test]
|
|
fn repack_keeps_flat_top_level_directories_under_the_prefix() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("flat-with-dir.tar.gz");
|
|
{
|
|
let file = std::fs::File::create(&source).unwrap();
|
|
let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::fast());
|
|
let mut builder = tar::Builder::new(encoder);
|
|
append_flat_dir(&mut builder, "src");
|
|
append_flat_file(&mut builder, "file1", "one\n");
|
|
append_flat_file(&mut builder, "src/main.rs", "code\n");
|
|
builder.into_inner().unwrap();
|
|
}
|
|
|
|
let dest = repack_tarball_file(&source, "mytool", "2.0.0", &tree).unwrap();
|
|
let names = tarball_names(&dest);
|
|
assert!(names.iter().any(|n| n == "mytool-2.0.0/src"), "{names:?}");
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-2.0.0/src/main.rs"),
|
|
"{names:?}"
|
|
);
|
|
assert!(names.iter().any(|n| n == "mytool-2.0.0/file1"), "{names:?}");
|
|
assert!(
|
|
names.iter().all(|n| n.starts_with("mytool-2.0.0/")),
|
|
"{names:?}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn repack_drops_git_dirs_and_unsupported_entries() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
// A gz tarball carrying a .git directory next to real sources.
|
|
let source = dir.path().join("old.tar.gz");
|
|
write_gz_fixture(
|
|
&source,
|
|
&[(".git/config", "ignored"), ("src/lib.rs", "code")],
|
|
);
|
|
|
|
let dest = repack_tarball_file(&source, "mytool", "0.1.0", &tree).unwrap();
|
|
let names = tarball_names(&dest);
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-0.1.0/src/lib.rs"),
|
|
"{names:?}"
|
|
);
|
|
assert!(!names.iter().any(|n| n.contains(".git")), "{names:?}");
|
|
}
|
|
|
|
#[test]
|
|
fn repack_rejects_unknown_extensions_and_existing_dest() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("upstream.tar.zip");
|
|
std::fs::write(&source, b"zip").unwrap();
|
|
let err = repack_tarball_file(&source, "mytool", "0.1.0", &tree).unwrap_err();
|
|
assert!(
|
|
err.to_string().contains("does not look like a tarball"),
|
|
"{err}"
|
|
);
|
|
|
|
// An existing destination is refused before anything is unpacked.
|
|
let source = dir.path().join("upstream.tar");
|
|
std::fs::write(&source, b"").unwrap();
|
|
std::fs::write(dir.path().join("mytool_0.1.0.orig.tar.xz"), b"old").unwrap();
|
|
let err = repack_tarball_file(&source, "mytool", "0.1.0", &tree).unwrap_err();
|
|
assert!(err.to_string().contains("already exists"), "{err}");
|
|
}
|
|
|
|
#[test]
|
|
fn repack_supports_plain_and_xz_tarballs() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
// Plain .tar.
|
|
let source = dir.path().join("upstream.tar");
|
|
{
|
|
let mut builder = tar::Builder::new(std::fs::File::create(&source).unwrap());
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_size(3);
|
|
header.set_mode(0o644);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(&mut header, "oldpkg-1.0/f.txt", "abc".as_bytes())
|
|
.unwrap();
|
|
builder.finish().unwrap();
|
|
}
|
|
let dest = repack_tarball_file(&source, "mytool", "2.0", &tree).unwrap();
|
|
assert!(tarball_names(&dest).contains(&"mytool-2.0/f.txt".to_string()));
|
|
|
|
// .tar.xz.
|
|
let source = dir.path().join("upstream.tar.xz");
|
|
{
|
|
let file = std::fs::File::create(&source).unwrap();
|
|
let encoder = xz2::write::XzEncoder::new(file, 1);
|
|
let mut builder = tar::Builder::new(encoder);
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_size(3);
|
|
header.set_mode(0o644);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(&mut header, "oldpkg-1.0/g.txt", "xyz".as_bytes())
|
|
.unwrap();
|
|
builder.finish().unwrap();
|
|
}
|
|
let dest = repack_tarball_file(&source, "mytool", "2.1", &tree).unwrap();
|
|
assert!(tarball_names(&dest).contains(&"mytool-2.1/g.txt".to_string()));
|
|
}
|
|
|
|
/// Every tarball format is recognized from its real content alone:
|
|
/// gzip (`1f 8b`), xz (`fd 37 7a 58 5a 00`), bzip2 (`"BZh"`) and
|
|
/// plain tar (`ustar` at offset 257). Garbage and truncated files
|
|
/// match nothing.
|
|
#[test]
|
|
fn sniff_detects_every_magic_signature() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
|
|
let gz = dir.path().join("f.tar.gz");
|
|
write_gz_fixture(&gz, &[("a.txt", "A")]);
|
|
assert_eq!(sniff_tarball_format(&gz), Some(TarballFormat::Gzip));
|
|
|
|
let xz = dir.path().join("f.tar.xz");
|
|
write_xz_fixture(&xz, &[("a.txt", "A")]);
|
|
assert_eq!(sniff_tarball_format(&xz), Some(TarballFormat::Xz));
|
|
|
|
// Real bzip2 bytes when the host binary exists, else the raw
|
|
// "BZh" header the sniffer keys on.
|
|
let bz2 = dir.path().join("f.tar.bz2");
|
|
if write_bzip2_fixture(&bz2, &[("a.txt", "A")]) {
|
|
assert_eq!(sniff_tarball_format(&bz2), Some(TarballFormat::Bzip2));
|
|
} else {
|
|
std::fs::write(&bz2, b"BZh9\x31\x41\x59\x26\x53\x59").unwrap();
|
|
assert_eq!(sniff_tarball_format(&bz2), Some(TarballFormat::Bzip2));
|
|
}
|
|
|
|
let tar = dir.path().join("f.tar");
|
|
write_plain_tar_fixture(&tar, &[("a.txt", "A")]);
|
|
assert_eq!(sniff_tarball_format(&tar), Some(TarballFormat::PlainTar));
|
|
|
|
// Garbage (a zip header, notably) matches nothing.
|
|
let garbage = dir.path().join("garbage.bin");
|
|
std::fs::write(&garbage, b"PK\x03\x04 definitely not a tarball").unwrap();
|
|
assert_eq!(sniff_tarball_format(&garbage), None);
|
|
// Files too short to even hold a signature.
|
|
let tiny = dir.path().join("tiny.bin");
|
|
std::fs::write(&tiny, b"no").unwrap();
|
|
assert_eq!(sniff_tarball_format(&tiny), None);
|
|
// Missing files are simply unrecognized.
|
|
assert_eq!(sniff_tarball_format(&dir.path().join("missing")), None);
|
|
}
|
|
|
|
/// The extension fallback (for local user-provided files whose content
|
|
/// carries no recognizable magic) maps every supported name, including
|
|
/// the short spellings, case-insensitively.
|
|
#[test]
|
|
fn extension_fallback_mapping() {
|
|
fn p(name: &str) -> &Path {
|
|
Path::new(name)
|
|
}
|
|
assert_eq!(
|
|
format_from_extension(p("a.tar.gz")),
|
|
Some(TarballFormat::Gzip)
|
|
);
|
|
assert_eq!(format_from_extension(p("a.TGZ")), Some(TarballFormat::Gzip));
|
|
assert_eq!(
|
|
format_from_extension(p("a.tar.xz")),
|
|
Some(TarballFormat::Xz)
|
|
);
|
|
assert_eq!(format_from_extension(p("a.txz")), Some(TarballFormat::Xz));
|
|
assert_eq!(
|
|
format_from_extension(p("a.tar.bz2")),
|
|
Some(TarballFormat::Bzip2)
|
|
);
|
|
assert_eq!(
|
|
format_from_extension(p("a.tbz2")),
|
|
Some(TarballFormat::Bzip2)
|
|
);
|
|
assert_eq!(
|
|
format_from_extension(p("a.tar")),
|
|
Some(TarballFormat::PlainTar)
|
|
);
|
|
assert_eq!(format_from_extension(p("a.tar.zip")), None);
|
|
assert_eq!(format_from_extension(p("a")), None);
|
|
}
|
|
|
|
/// Regression (release-download origin): a downloaded stream lands in
|
|
/// an EXTENSIONLESS temporary file, which the extension-based check
|
|
/// always rejected ('does not look like a tarball'), silently falling
|
|
/// through to `git archive`. Valid gzip and plain-tar content must
|
|
/// round-trip through such a name.
|
|
#[test]
|
|
fn repack_sniffs_extensionless_downloaded_tarballs() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
// A gzip stream, like a codeload tar.gz download.
|
|
let source = dir.path().join("pkh-orig-1234");
|
|
write_gz_fixture(&source, &[("src/main.rs", "hi\n")]);
|
|
let dest = repack_tarball_file(&source, "mytool", "1.0.0", &tree).unwrap();
|
|
assert!(tarball_names(&dest).contains(&"mytool-1.0.0/src/main.rs".to_string()));
|
|
|
|
// A plain tar stream, also extensionless.
|
|
let source = dir.path().join("pkh-orig-5678");
|
|
write_plain_tar_fixture(&source, &[("README", "readme\n")]);
|
|
let dest = repack_tarball_file(&source, "mytool", "1.0.1", &tree).unwrap();
|
|
assert!(tarball_names(&dest).contains(&"mytool-1.0.1/README".to_string()));
|
|
}
|
|
|
|
/// The sniffed decompression route also covers xz (and bzip2, below):
|
|
/// a valid xz stream round-trips through an extensionless name.
|
|
#[test]
|
|
fn repack_sniffs_xz_tarballs() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("pkh-orig-xz");
|
|
write_xz_fixture(&source, &[("x.txt", "X\n")]);
|
|
let dest = repack_tarball_file(&source, "mytool", "1.1.0", &tree).unwrap();
|
|
assert!(tarball_names(&dest).contains(&"mytool-1.1.0/x.txt".to_string()));
|
|
}
|
|
|
|
/// bzip2 content is sniffed from its "BZh" header and still
|
|
/// decompressed through the host `bzip2` binary, extensionless or not.
|
|
#[test]
|
|
fn repack_sniffs_bzip2_tarballs() {
|
|
if !std::process::Command::new("bzip2")
|
|
.arg("--version")
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
{
|
|
return;
|
|
}
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("pkh-orig-bz2");
|
|
assert!(write_bzip2_fixture(&source, &[("b.txt", "B\n")]));
|
|
let dest = repack_tarball_file(&source, "mytool", "1.2.0", &tree).unwrap();
|
|
assert!(tarball_names(&dest).contains(&"mytool-1.2.0/b.txt".to_string()));
|
|
}
|
|
|
|
/// Content sniffing takes precedence over the name: a gzip stream
|
|
/// misnamed '.tar.xz' is decoded as gzip, not xz.
|
|
#[test]
|
|
fn repack_content_sniffing_beats_the_extension() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("misnamed.tar.xz");
|
|
write_gz_fixture(&source, &[("f.txt", "x\n")]);
|
|
let dest = repack_tarball_file(&source, "mytool", "1.3.0", &tree).unwrap();
|
|
assert!(tarball_names(&dest).contains(&"mytool-1.3.0/f.txt".to_string()));
|
|
}
|
|
|
|
/// Unsniffable content is rejected with a clear error whether or not
|
|
/// the name carries an extension — a download can only fail on
|
|
/// unrecognized CONTENT, never on a missing extension.
|
|
#[test]
|
|
fn repack_rejects_unsniffable_content() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
// Extensionless, like a downloaded temp file.
|
|
let source = dir.path().join("pkh-orig-0000");
|
|
std::fs::write(&source, b"PK\x03\x04 zip data").unwrap();
|
|
let err = repack_tarball_file(&source, "mytool", "9.9.9", &tree).unwrap_err();
|
|
assert!(
|
|
err.to_string().contains("does not look like a tarball"),
|
|
"{err}"
|
|
);
|
|
// No half-written tarball is left behind.
|
|
assert!(!dir.path().join("mytool_9.9.9.orig.tar.xz").exists());
|
|
}
|
|
|
|
#[test]
|
|
fn repack_preserves_the_exec_bit() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("upstream.tar");
|
|
{
|
|
let mut builder = tar::Builder::new(std::fs::File::create(&source).unwrap());
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_size(11);
|
|
header.set_mode(0o755);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(
|
|
&mut header,
|
|
"oldpkg-1.0/run.sh",
|
|
"#!/bin/sh\nx\n".as_bytes(),
|
|
)
|
|
.unwrap();
|
|
builder.finish().unwrap();
|
|
}
|
|
let dest = repack_tarball_file(&source, "mytool", "0.5.0", &tree).unwrap();
|
|
let mut archive = tar::Archive::new(xz2::read::XzDecoder::new(
|
|
std::fs::File::open(&dest).unwrap(),
|
|
));
|
|
for entry in archive.entries().unwrap() {
|
|
let entry = entry.unwrap();
|
|
if entry.path().unwrap().ends_with("run.sh") {
|
|
assert_eq!(entry.header().mode().unwrap() & 0o111, 0o111);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn vendor_component_layout_and_overwrite_refusal() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(tree.join("vendor/serde/src")).unwrap();
|
|
std::fs::write(tree.join("vendor/serde/src/lib.rs"), "crate code").unwrap();
|
|
std::fs::write(tree.join("vendor/serde/Cargo.toml"), "[package]").unwrap();
|
|
|
|
let component = create_vendor_component(&tree, "mytool", "1.0.0").unwrap();
|
|
assert_eq!(
|
|
component,
|
|
dir.path().join("mytool_1.0.0.orig-vendor.tar.xz")
|
|
);
|
|
let names = tarball_names(&component);
|
|
assert!(
|
|
names.iter().any(|n| n == "vendor/serde/src/lib.rs"),
|
|
"{names:?}"
|
|
);
|
|
// The top-level entry is the bare `vendor/` directory.
|
|
assert!(
|
|
names.iter().any(|n| n.trim_end_matches('/') == "vendor"),
|
|
"{names:?}"
|
|
);
|
|
|
|
// A second creation refuses to overwrite the stale component.
|
|
let err = create_vendor_component(&tree, "mytool", "1.0.0").unwrap_err();
|
|
assert!(err.to_string().contains("already exists"), "{err}");
|
|
}
|
|
|
|
#[test]
|
|
fn vendor_component_needs_a_non_empty_vendor_dir() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
// No vendor/ at all.
|
|
let err = create_vendor_component(&tree, "mytool", "1.0.0").unwrap_err();
|
|
assert!(err.to_string().contains("vendor"), "{err}");
|
|
// An empty vendor/ counts as nothing.
|
|
std::fs::create_dir_all(tree.join("vendor")).unwrap();
|
|
assert!(create_vendor_component(&tree, "mytool", "1.0.0").is_err());
|
|
}
|
|
|
|
/// Regression (pkh build re-vendor retry): the component file name is
|
|
/// derived from the changelog version's UPSTREAM part, never the full
|
|
/// version. The retry hook must therefore compute
|
|
/// `<name>_0.14.0.orig-vendor.tar.xz` for changelog version
|
|
/// `0.14.0-1` — the exact file `create_vendor_component` names — or a
|
|
/// stale component silently survives the recreation.
|
|
#[test]
|
|
fn component_name_uses_the_upstream_version_part() {
|
|
use crate::debian::DebianVersion;
|
|
|
|
// `0.14.0-1`: upstream part only. NOT `0.14.0-1`
|
|
// (`DebianVersion::no_epoch()`), which was the original bug.
|
|
let version = DebianVersion::parse("0.14.0-1").unwrap();
|
|
assert_eq!(component_upstream_version(&version), "0.14.0");
|
|
// Epochs are stripped the same way.
|
|
let epochy = DebianVersion::parse("2:0.14.0-1").unwrap();
|
|
assert_eq!(component_upstream_version(&epochy), "0.14.0");
|
|
|
|
// End-to-end naming: the path the retry hook looks up and the file
|
|
// `create_vendor_component` writes are one and the same.
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(tree.join("vendor/serde")).unwrap();
|
|
std::fs::write(tree.join("vendor/serde/lib.rs"), "code").unwrap();
|
|
let created =
|
|
create_vendor_component(&tree, "mytool", component_upstream_version(&version)).unwrap();
|
|
assert_eq!(
|
|
created,
|
|
vendor_component_path(&tree, "mytool", component_upstream_version(&version)).unwrap()
|
|
);
|
|
assert_eq!(
|
|
created.file_name().unwrap(),
|
|
std::ffi::OsStr::new("mytool_0.14.0.orig-vendor.tar.xz")
|
|
);
|
|
// No revision-suffixed variant may exist next to it.
|
|
assert!(
|
|
!dir.path()
|
|
.join("mytool_0.14.0-1.orig-vendor.tar.xz")
|
|
.exists()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn has_vendored_dir_detection() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
assert!(!has_vendored_dir(&tree));
|
|
std::fs::create_dir_all(tree.join("vendor")).unwrap();
|
|
assert!(!has_vendored_dir(&tree)); // empty
|
|
std::fs::write(tree.join("vendor/x"), "y").unwrap();
|
|
assert!(has_vendored_dir(&tree));
|
|
}
|
|
|
|
/// The snapshot origin with `vendored_rust` excludes the top-level
|
|
/// `vendor/` from the main orig (that is what the component is for).
|
|
#[test]
|
|
fn snapshot_origin_excludes_vendor_when_vendored() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(tree.join("vendor/serde/src")).unwrap();
|
|
std::fs::write(tree.join("vendor/serde/src/lib.rs"), "code").unwrap();
|
|
std::fs::write(tree.join("Cargo.toml"), "[package]").unwrap();
|
|
|
|
let created = create_orig(&tree, "mytool", "1.0.0", &OrigOrigin::Snapshot, true).unwrap();
|
|
assert_eq!(created.label, "working tree snapshot");
|
|
let names = tarball_names(&created.path);
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-1.0.0/Cargo.toml"),
|
|
"{names:?}"
|
|
);
|
|
assert!(!names.iter().any(|n| n.contains("vendor")), "{names:?}");
|
|
|
|
// Without the vendored-rust marker the directory stays in (a
|
|
// non-rust project may legitimately carry one).
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(tree.join("src/vendor")).unwrap();
|
|
std::fs::write(tree.join("src/vendor/mod.rs"), "code").unwrap();
|
|
let created = create_orig(&tree, "mytool", "1.0.0", &OrigOrigin::Snapshot, false).unwrap();
|
|
let names = tarball_names(&created.path);
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-1.0.0/src/vendor/mod.rs"),
|
|
"{names:?}"
|
|
);
|
|
}
|
|
|
|
/// git archive origin: a scripted repo with a tag produces exactly the
|
|
/// tagged content under the `<name>-<uver>/` prefix.
|
|
#[test]
|
|
fn git_archive_origin_packs_the_tag_content() {
|
|
if std::process::Command::new("git")
|
|
.arg("--version")
|
|
.output()
|
|
.is_err()
|
|
{
|
|
return;
|
|
}
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let repo = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&repo).unwrap();
|
|
let git = |args: &[&str]| {
|
|
let status = std::process::Command::new("git")
|
|
.args([
|
|
"-c",
|
|
"user.name=T",
|
|
"-c",
|
|
"user.email=t@example.invalid",
|
|
"-c",
|
|
"commit.gpgsign=false",
|
|
])
|
|
.args(args)
|
|
.current_dir(&repo)
|
|
.status()
|
|
.unwrap();
|
|
assert!(status.success(), "git {args:?} failed");
|
|
};
|
|
git(&["init", "-q"]);
|
|
std::fs::write(repo.join("hello.txt"), "release\n").unwrap();
|
|
git(&["add", "hello.txt"]);
|
|
git(&["commit", "-q", "-m", "release"]);
|
|
git(&["tag", "v1.2.3"]);
|
|
// A later, uncommitted-looking file exists in the worktree but must
|
|
// NOT travel into the tag archive.
|
|
std::fs::write(repo.join("uncommitted.txt"), "dirty\n").unwrap();
|
|
|
|
let created = create_orig(
|
|
&repo,
|
|
"mytool",
|
|
"1.2.3",
|
|
&OrigOrigin::GitArchive {
|
|
tag: "v1.2.3".to_string(),
|
|
},
|
|
false,
|
|
)
|
|
.unwrap();
|
|
assert_eq!(created.label, "git archive (v1.2.3)");
|
|
let names = tarball_names(&created.path);
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-1.2.3/hello.txt"),
|
|
"{names:?}"
|
|
);
|
|
assert!(
|
|
!names.iter().any(|n| n.contains("uncommitted")),
|
|
"{names:?}"
|
|
);
|
|
}
|
|
|
|
/// A failing git archive (tag missing) is reported, not silently
|
|
/// swallowed.
|
|
#[test]
|
|
fn git_archive_origin_fails_on_a_missing_tag() {
|
|
if std::process::Command::new("git")
|
|
.arg("--version")
|
|
.output()
|
|
.is_err()
|
|
{
|
|
return;
|
|
}
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let repo = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&repo).unwrap();
|
|
let status = std::process::Command::new("git")
|
|
.args(["init", "-q"])
|
|
.current_dir(&repo)
|
|
.status()
|
|
.unwrap();
|
|
assert!(status.success());
|
|
|
|
let err = create_orig(
|
|
&repo,
|
|
"mytool",
|
|
"1.0.0",
|
|
&OrigOrigin::GitArchive {
|
|
tag: "v9.9.9".to_string(),
|
|
},
|
|
false,
|
|
)
|
|
.unwrap_err();
|
|
assert!(err.to_string().contains("git archive"), "{err}");
|
|
// No half-written tarball is left behind.
|
|
assert!(!dir.path().join("mytool_1.0.0.orig.tar.xz").exists());
|
|
}
|
|
|
|
/// The release origin falls through to `git archive` when the download
|
|
/// fails (offline host): the label reflects the actual origin.
|
|
#[test]
|
|
fn release_origin_falls_through_to_git_archive() {
|
|
if std::process::Command::new("git")
|
|
.arg("--version")
|
|
.output()
|
|
.is_err()
|
|
{
|
|
return;
|
|
}
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let repo = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&repo).unwrap();
|
|
let git = |args: &[&str]| {
|
|
let status = std::process::Command::new("git")
|
|
.args([
|
|
"-c",
|
|
"user.name=T",
|
|
"-c",
|
|
"user.email=t@example.invalid",
|
|
"-c",
|
|
"commit.gpgsign=false",
|
|
])
|
|
.args(args)
|
|
.current_dir(&repo)
|
|
.status()
|
|
.unwrap();
|
|
assert!(status.success(), "git {args:?} failed");
|
|
};
|
|
git(&["init", "-q"]);
|
|
std::fs::write(repo.join("f"), "x\n").unwrap();
|
|
git(&["add", "f"]);
|
|
git(&["commit", "-q", "-m", "first"]);
|
|
git(&["tag", "v0.1.0"]);
|
|
|
|
// An unreachable URL: the download fails quickly (connection
|
|
// refused on a reserved port), the git archive takes over.
|
|
let created = create_orig(
|
|
&repo,
|
|
"mytool",
|
|
"0.1.0",
|
|
&OrigOrigin::Release {
|
|
tag: "v0.1.0".to_string(),
|
|
forge: Forge::parse("https://github.com/pkh-nonexistent-org/pkh-nonexistent-repo")
|
|
.unwrap(),
|
|
},
|
|
false,
|
|
)
|
|
.unwrap();
|
|
assert_eq!(created.label, "git archive (v0.1.0)");
|
|
assert!(created.path.exists());
|
|
}
|
|
|
|
/// The provided origin accepts a local gz tarball and repacks it.
|
|
#[test]
|
|
fn provided_origin_repacks_a_local_tarball() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
|
|
let source = dir.path().join("given.tar.gz");
|
|
write_gz_fixture(&source, &[("main.rs", "fn main() {}\n")]);
|
|
|
|
let created = create_orig(
|
|
&tree,
|
|
"mytool",
|
|
"3.0.0",
|
|
&OrigOrigin::Provided {
|
|
source: source.to_string_lossy().into_owned(),
|
|
},
|
|
false,
|
|
)
|
|
.unwrap();
|
|
assert!(created.label.starts_with("user tarball"));
|
|
let names = tarball_names(&created.path);
|
|
assert!(
|
|
names.iter().any(|n| n == "mytool-3.0.0/main.rs"),
|
|
"{names:?}"
|
|
);
|
|
}
|
|
|
|
/// Sanity: a tar builder accepts empty writes for directory headers
|
|
/// (the repack path relies on it).
|
|
#[test]
|
|
fn empty_write_directory_header_roundtrip() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let dest = dir.path().join("t.tar");
|
|
let mut builder = tar::Builder::new(std::fs::File::create(&dest).unwrap());
|
|
let mut header = tar::Header::new_gnu();
|
|
header.set_entry_type(tar::EntryType::Directory);
|
|
header.set_size(0);
|
|
header.set_mode(0o755);
|
|
header.set_cksum();
|
|
builder
|
|
.append_data(&mut header, "top", std::io::empty())
|
|
.unwrap();
|
|
builder.finish().unwrap();
|
|
let mut archive = tar::Archive::new(std::fs::File::open(&dest).unwrap());
|
|
let names: Vec<String> = archive
|
|
.entries()
|
|
.unwrap()
|
|
.map(|e| e.unwrap().path().unwrap().to_string_lossy().into_owned())
|
|
.collect();
|
|
assert_eq!(names, vec!["top"]);
|
|
}
|
|
|
|
/// Ensure writes into the gz fixture builder produce a readable tarball
|
|
/// (trips on header size mismatches).
|
|
#[test]
|
|
fn gz_fixture_builder_produces_readable_tarballs() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let source = dir.path().join("fixture.tar.gz");
|
|
write_gz_fixture(&source, &[("a.txt", "A")]);
|
|
let decoder = flate2::read::GzDecoder::new(std::fs::File::open(&source).unwrap());
|
|
let mut archive = tar::Archive::new(decoder);
|
|
let mut seen = Vec::new();
|
|
for entry in archive.entries().unwrap() {
|
|
let mut entry = entry.unwrap();
|
|
let name = entry.path().unwrap().to_string_lossy().into_owned();
|
|
seen.push(name.clone());
|
|
if name.ends_with('/') || entry.header().entry_type() == tar::EntryType::Directory {
|
|
continue;
|
|
}
|
|
let mut contents = String::new();
|
|
entry.read_to_string(&mut contents).unwrap();
|
|
assert_eq!(contents, "A");
|
|
}
|
|
assert!(seen.contains(&"oldpkg-1.0/a.txt".to_string()), "{seen:?}");
|
|
}
|
|
|
|
/// Ensure writes into the xz encoder fail loudly when the stream is
|
|
/// not a tar at all (garbage input produces a readable error path).
|
|
#[test]
|
|
fn repack_garbage_fails_cleanly() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let tree = dir.path().join("mytool");
|
|
std::fs::create_dir_all(&tree).unwrap();
|
|
let source = dir.path().join("garbage.tar.gz");
|
|
{
|
|
let mut file = std::fs::File::create(&source).unwrap();
|
|
file.write_all(b"definitely not gzip").unwrap();
|
|
}
|
|
let result = repack_tarball_file(&source, "mytool", "1.0.0", &tree);
|
|
// Either the gzip header check fails immediately or no tarball is
|
|
// left over — both are acceptable failures, a corrupt orig is not.
|
|
if let Ok(dest) = result {
|
|
assert!(
|
|
tarball_names(&dest).is_empty() || !dest.exists(),
|
|
"garbage input must not produce a usable orig"
|
|
);
|
|
}
|
|
assert!(!dir.path().join("mytool_1.0.0.orig.tar.xz").exists());
|
|
}
|
|
}
|