vhaudiquet
ae679f29d9
The pull downloads reported through the plain count port only because
byte_progress landed on main after the branch was cut. PullView now
implements it with a byte-counter template in the transfer-style
family (same shape, {bytes}/{total_bytes} counters, human-formatted by
indicatif), downloads emit through it, and the trait doc lists the
event among the ones the pull flow emits — un-duplicating the merged
'message, progress and progress' sentence.
1532 lines
56 KiB
Rust
1532 lines
56 KiB
Rust
use std::cmp::min;
|
|
use std::error::Error;
|
|
use std::os::unix::fs::symlink;
|
|
use std::path::Component;
|
|
use std::path::Path;
|
|
use std::path::PathBuf;
|
|
|
|
use crate::package_info::PackageInfo;
|
|
|
|
use std::process::Command;
|
|
|
|
use log::debug;
|
|
|
|
use regex::Regex;
|
|
|
|
use crate::report::BuildView;
|
|
|
|
fn clone_repo(
|
|
url: &str,
|
|
package: &str,
|
|
branch: Option<&str>,
|
|
cwd: Option<&Path>,
|
|
view: &dyn BuildView,
|
|
) -> Result<(), Box<dyn Error>> {
|
|
let target_path = if let Some(path) = cwd {
|
|
path.join(package)
|
|
} else {
|
|
Path::new(package).to_path_buf()
|
|
};
|
|
|
|
let mut callbacks = git2::RemoteCallbacks::new();
|
|
// The sideband parser is only worth running when something listens
|
|
if view.is_enabled() {
|
|
// Download progress
|
|
callbacks.transfer_progress(move |stats| {
|
|
view.progress(
|
|
"Receiving objects...",
|
|
stats.received_objects(),
|
|
stats.total_objects(),
|
|
);
|
|
true
|
|
});
|
|
|
|
// Remote progress: messages 'Remote: compressing objects 10% (34/340)'
|
|
// Parse progress informations to display them in the view
|
|
callbacks.sideband_progress(move |data| {
|
|
let msg = String::from_utf8_lossy(data);
|
|
let re = Regex::new(r"(.*):[ ]*([0-9]*)% \(([0-9]*)/([0-9]*)\)").unwrap();
|
|
if let Some(caps) = re.captures(msg.trim()) {
|
|
let msg = caps.get(1).map_or("", |m| m.as_str()).to_string();
|
|
let objects = caps
|
|
.get(3)
|
|
.map_or("", |m| m.as_str())
|
|
.to_string()
|
|
.parse::<usize>()
|
|
.unwrap_or(0);
|
|
let total = caps
|
|
.get(4)
|
|
.map_or("", |m| m.as_str())
|
|
.to_string()
|
|
.parse::<usize>()
|
|
.unwrap_or(0);
|
|
|
|
view.progress(msg.as_str(), objects, total);
|
|
}
|
|
|
|
true
|
|
});
|
|
}
|
|
|
|
let mut fetch_options = git2::FetchOptions::new();
|
|
fetch_options.remote_callbacks(callbacks);
|
|
|
|
let mut builder = git2::build::RepoBuilder::new();
|
|
builder.fetch_options(fetch_options);
|
|
|
|
if let Some(b) = branch {
|
|
builder.branch(b);
|
|
}
|
|
|
|
match builder.clone(url, &target_path) {
|
|
Ok(_repo) => Ok(()),
|
|
Err(e) => Err(format!("Failed to clone: {}", e).into()),
|
|
}
|
|
}
|
|
|
|
use std::fs::File;
|
|
use std::io::Write;
|
|
|
|
use flate2::read::GzDecoder;
|
|
use futures_util::StreamExt;
|
|
use tar::Archive;
|
|
use xz2::read::XzDecoder;
|
|
|
|
fn copy_dir_all(src: &Path, dst: &Path) -> Result<(), Box<dyn Error>> {
|
|
if !dst.exists() {
|
|
std::fs::create_dir_all(dst)?;
|
|
}
|
|
|
|
for entry in std::fs::read_dir(src)? {
|
|
let entry = entry?;
|
|
let src_path = entry.path();
|
|
let dst_path = dst.join(entry.file_name());
|
|
|
|
// Reproduce symlinks as symlinks rather than following them, so that
|
|
// dangling/absolute symlinks do not abort the copy.
|
|
if std::fs::symlink_metadata(&src_path)?
|
|
.file_type()
|
|
.is_symlink()
|
|
{
|
|
let target = std::fs::read_link(&src_path)?;
|
|
let _ = std::fs::remove_file(&dst_path);
|
|
symlink(&target, &dst_path)?;
|
|
} else if src_path.is_dir() {
|
|
copy_dir_all(&src_path, &dst_path)?;
|
|
} else {
|
|
// Copy the file, preserving the source modification and access
|
|
// times. This is important for autotools/gnulib-based packages
|
|
// (e.g. 'hello' from Debian sid) that ship pre-generated files
|
|
// alongside their prerequisites: if the copy resets the mtime to
|
|
// "now", the prerequisites appear as new as the generated targets
|
|
// and `make` tries to regenerate them using tools (like gperf)
|
|
// that are not declared build-dependencies.
|
|
std::fs::copy(&src_path, &dst_path)?;
|
|
copy_file_times(&src_path, &dst_path)?;
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Restore the modification and access times of `dest` to match `src`.
|
|
///
|
|
/// `std::fs::copy` resets the destination mtime to "now", which breaks
|
|
/// timestamp-based build systems (autotools/gnulib) that ship pre-generated
|
|
/// files alongside their prerequisites. Restoring the original timestamps
|
|
/// prevents `make` from needlessly regenerating those files with tools that
|
|
/// may not be installed (e.g. `gperf`).
|
|
fn copy_file_times(src: &Path, dest: &Path) -> Result<(), Box<dyn Error>> {
|
|
let metadata = std::fs::metadata(src)?;
|
|
let mut times = std::fs::FileTimes::new();
|
|
let mut have_times = false;
|
|
if let Ok(mtime) = metadata.modified() {
|
|
times = times.set_modified(mtime);
|
|
have_times = true;
|
|
}
|
|
if let Ok(atime) = metadata.accessed() {
|
|
times = times.set_accessed(atime);
|
|
have_times = true;
|
|
}
|
|
if have_times && let Ok(dest_file) = std::fs::File::open(dest) {
|
|
let _ = dest_file.set_times(times);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// The addon name of an extra upstream component tarball ('pigeonhole' for
|
|
/// 'pkg_1.0.orig-pigeonhole.tar.gz'): dpkg-source unpacks it into
|
|
/// '<root>/<addon>'. None for the main orig and every non-component file.
|
|
fn component_addon(name: &str) -> Option<&str> {
|
|
let rest = name.split_once(".orig-")?.1;
|
|
let addon = rest.split(".tar.").next()?;
|
|
if addon.is_empty() { None } else { Some(addon) }
|
|
}
|
|
|
|
/// Move the contents of an archive's wrapper directory into `dir`, or into
|
|
/// `dir/<addon>` for a multi-orig component, replacing any pre-existing
|
|
/// addon directory.
|
|
fn unwrap_into(dir: &Path, wrapper: &Path, addon: Option<&str>) -> Result<(), Box<dyn Error>> {
|
|
match addon {
|
|
// dpkg-source unpacks a component into '<root>/<addon>': a tarball
|
|
// whose entries already carry that prefix ('www/...' in
|
|
// 'sqlite3_3.53.4.orig-www.tar.xz') extracts to its end state, and
|
|
// unwrapping it would clear the target and move the cleared
|
|
// directory into itself.
|
|
Some(addon) if wrapper.file_name().is_some_and(|name| name == addon) => Ok(()),
|
|
Some(addon) => {
|
|
let target = dir.join(addon);
|
|
if target.is_dir() {
|
|
std::fs::remove_dir_all(&target)?;
|
|
}
|
|
std::fs::create_dir_all(&target)?;
|
|
unwrap_wrapper(&target, wrapper)
|
|
}
|
|
None => unwrap_wrapper(dir, wrapper),
|
|
}
|
|
}
|
|
|
|
/// Move the contents of an archive's wrapper directory up into `dir` itself.
|
|
///
|
|
/// Upstream tarballs are usually wrapped in a 'package-version/' directory
|
|
/// while the tree root is `dir`. The wrapper's entries are moved up one
|
|
/// level by rename — symlinks and timestamps survive a move, while a copy
|
|
/// flattens links and resets times — and the emptied wrapper is removed.
|
|
/// A name already present at the root (a multi-orig component laid files
|
|
/// there before) is merged over rather than clobbered.
|
|
fn unwrap_wrapper(dir: &Path, wrapper: &Path) -> Result<(), Box<dyn Error>> {
|
|
debug!("Unwrapping {} into {}", wrapper.display(), dir.display());
|
|
for entry in std::fs::read_dir(wrapper)? {
|
|
let entry = entry?;
|
|
let target = dir.join(entry.file_name());
|
|
if !target.exists() {
|
|
std::fs::rename(entry.path(), &target)?;
|
|
} else {
|
|
// The tree root already carries that name (a multi-orig archive
|
|
// laid files there before): merge over it instead of clobbering
|
|
if entry.path().is_dir() {
|
|
std::fs::create_dir_all(&target)?;
|
|
copy_dir_all(&entry.path(), &target)?;
|
|
std::fs::remove_dir_all(entry.path())?;
|
|
} else {
|
|
std::fs::copy(entry.path(), &target)?;
|
|
copy_file_times(&entry.path(), &target)?;
|
|
std::fs::remove_file(entry.path())?;
|
|
}
|
|
}
|
|
}
|
|
std::fs::remove_dir(wrapper)?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Result of extracting an archive
|
|
struct ExtractedArchive {
|
|
/// The single first path component every entry of the archive shares:
|
|
/// its 'package-version/' wrapper, to unwrap by the caller. `None` when
|
|
/// the archive is already laid out in place (several first components,
|
|
/// root-level files, an empty archive).
|
|
wrapper: Option<PathBuf>,
|
|
}
|
|
|
|
/// Helper function to extract tar archive with progress tracking
|
|
fn extract_tar_archive<D, F>(
|
|
file_path: &Path,
|
|
dest: &Path,
|
|
view: &dyn BuildView,
|
|
decoder_factory: F,
|
|
) -> Result<ExtractedArchive, Box<dyn Error>>
|
|
where
|
|
D: std::io::Read,
|
|
F: Fn(File) -> D,
|
|
{
|
|
let file = File::open(file_path)?;
|
|
let decoder = decoder_factory(file);
|
|
let mut archive = Archive::new(decoder);
|
|
|
|
// Get total number of entries for progress tracking
|
|
let total_entries = archive.entries()?.count();
|
|
let mut current_entry = 0;
|
|
|
|
// Reset the archive to read entries again
|
|
let file = File::open(file_path)?;
|
|
let decoder = decoder_factory(file);
|
|
let mut archive = Archive::new(decoder);
|
|
|
|
// The archive's wrapper: the one first path component every entry
|
|
// shares. Tracked from the entries themselves — tarballs are not
|
|
// required to carry entries for their directories, and a missing entry
|
|
// used to leave the sources nested one level too deep.
|
|
let mut root: Option<std::ffi::OsString> = None;
|
|
let mut multiple_roots = false;
|
|
|
|
for entry in archive.entries()? {
|
|
let mut entry = entry?;
|
|
|
|
// A pax global header record ('pax_global_header', the opening
|
|
// entry of git-archive tarballs) is metadata about the archive,
|
|
// not content: GNU tar consumes it and never lists or extracts
|
|
// it, but the tar crate yields it as an entry. It must not be
|
|
// extracted (unpack would no-op) and must not count for the
|
|
// wrapper detection below — it used to make every git-archive
|
|
// orig look multi-rooted, skipping the unwrap and leaving the
|
|
// sources nested one level too deep.
|
|
if entry.header().entry_type().is_pax_global_extensions() {
|
|
continue;
|
|
}
|
|
|
|
let path = entry.path()?.to_path_buf();
|
|
|
|
// Archives built with './'-prefixed entries (common in third-party
|
|
// repositories) target the destination directory itself, with no
|
|
// 'package-version/' wrapper; strip the prefix so entries land there
|
|
// too, and skip their root entry
|
|
let relative = path.strip_prefix("./").unwrap_or(&path).to_path_buf();
|
|
|
|
if relative.as_os_str().is_empty() {
|
|
continue;
|
|
}
|
|
|
|
let first = relative.components().next();
|
|
if let Some(Component::Normal(name)) = first {
|
|
match root.take() {
|
|
None => root = Some(name.to_os_string()),
|
|
Some(shared) if shared.as_os_str() == name => root = Some(shared),
|
|
Some(_) => multiple_roots = true,
|
|
}
|
|
}
|
|
|
|
// Security: `Entry::unpack` performs no path sanitization, so extract
|
|
// the entry only if its path is confined to the destination directory.
|
|
// An absolute path or a path containing '..' would let a malicious (or
|
|
// deeply malformed) tarball write files anywhere outside 'dest' (path
|
|
// traversal). Refuse such entries with an error rather than skipping
|
|
// them silently, so the problem is not hidden.
|
|
let escapes_dest = relative.components().any(|component| {
|
|
matches!(
|
|
component,
|
|
Component::Prefix(_) | Component::RootDir | Component::ParentDir
|
|
)
|
|
});
|
|
if escapes_dest {
|
|
return Err(format!(
|
|
"Refusing to extract '{}': archive entry path is absolute or \
|
|
contains '..' and would escape the destination directory '{}'",
|
|
relative.display(),
|
|
dest.display()
|
|
)
|
|
.into());
|
|
}
|
|
|
|
let dest_path = dest.join(&relative);
|
|
|
|
// Create parent directories if needed
|
|
if let Some(parent) = dest_path.parent() {
|
|
std::fs::create_dir_all(parent)?;
|
|
}
|
|
|
|
// A hard link shares its content with an earlier entry of the
|
|
// archive: tar stores the target path relative to the archive root,
|
|
// where the shared entry has already been extracted. Resolving it
|
|
// any other way (as `Entry::unpack` does) misses the target and
|
|
// aborts the whole extraction.
|
|
if entry.header().entry_type().is_hard_link() {
|
|
let link_target = entry.link_name()?.ok_or_else(|| {
|
|
format!(
|
|
"archive entry '{}' is a hard link without a link target",
|
|
relative.display()
|
|
)
|
|
})?;
|
|
// Absolute link targets land under the destination root, like
|
|
// GNU tar treats them; anything else escaping 'dest' is refused
|
|
let link_relative = link_target.strip_prefix("/").unwrap_or(&link_target);
|
|
let escapes_dest = link_relative.components().any(|component| {
|
|
matches!(
|
|
component,
|
|
Component::Prefix(_) | Component::RootDir | Component::ParentDir
|
|
)
|
|
});
|
|
if escapes_dest {
|
|
return Err(format!(
|
|
"Refusing to extract '{}': hard link target '{}' is \
|
|
absolute or contains '..' and would escape the \
|
|
destination directory '{}'",
|
|
relative.display(),
|
|
link_relative.display(),
|
|
dest.display()
|
|
)
|
|
.into());
|
|
}
|
|
let link_path = dest.join(link_relative);
|
|
std::fs::hard_link(&link_path, &dest_path).map_err(|e| {
|
|
format!(
|
|
"hard linking archive entry '{}' to its target '{}' \
|
|
failed: {e}",
|
|
relative.display(),
|
|
link_relative.display()
|
|
)
|
|
})?;
|
|
} else {
|
|
// Extract the entry
|
|
entry.unpack(&dest_path)?;
|
|
}
|
|
|
|
current_entry += 1;
|
|
|
|
// Report progress
|
|
view.progress("Extracting...", current_entry, total_entries);
|
|
}
|
|
|
|
let wrapper = if multiple_roots {
|
|
None
|
|
} else {
|
|
root.map(|r| dest.join(r))
|
|
};
|
|
Ok(ExtractedArchive { wrapper })
|
|
}
|
|
|
|
fn extract_archive(
|
|
path: &Path,
|
|
dest: &Path,
|
|
view: &dyn BuildView,
|
|
) -> Result<ExtractedArchive, Box<dyn Error>> {
|
|
let filename = path.file_name().unwrap().to_string_lossy();
|
|
|
|
if filename.ends_with(".tar.gz") || filename.ends_with(".tgz") {
|
|
extract_tar_archive(path, dest, view, GzDecoder::new)
|
|
} else if filename.ends_with(".tar.xz") || filename.ends_with(".txz") {
|
|
extract_tar_archive(path, dest, view, XzDecoder::new)
|
|
} else if filename.ends_with(".tar.bz2") || filename.ends_with(".tbz2") {
|
|
extract_tar_archive(path, dest, view, bzip2_rs::DecoderReader::new)
|
|
} else {
|
|
Err(format!("Unsupported archive format: {}", filename).into())
|
|
}
|
|
}
|
|
|
|
fn checkout_pristine_tar(package_dir: &Path, filename: &str) -> Result<(), Box<dyn Error>> {
|
|
let output = Command::new("pristine-tar")
|
|
.current_dir(package_dir)
|
|
.args(["checkout", format!("../{filename}").as_str()])
|
|
.output()
|
|
.map_err(|e| {
|
|
format!(
|
|
"Failed to run 'pristine-tar' to check out '{filename}': {}. \
|
|
Is 'pristine-tar' installed? It is required to reconstruct \
|
|
the upstream orig tarball from a git repository.",
|
|
e
|
|
)
|
|
})?;
|
|
|
|
if !output.status.success() {
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
return Err(format!(
|
|
"pristine-tar checkout of '{filename}' failed with status: {}.{}",
|
|
output.status,
|
|
if stderr.trim().is_empty() {
|
|
String::new()
|
|
} else {
|
|
format!("\npristine-tar output:\n{}", stderr.trim())
|
|
}
|
|
)
|
|
.into());
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
async fn download_file_checksum(
|
|
url: &str,
|
|
checksum: &str,
|
|
algo: crate::package_info::ChecksumAlgo,
|
|
target_dir: &Path,
|
|
view: &dyn BuildView,
|
|
) -> Result<(), Box<dyn Error>> {
|
|
// Archive mirrors and CDNs are busy enough that single attempts fail
|
|
// spuriously (dropped connections, truncated bodies, index generations
|
|
// momentarily out of sync): retry the whole download a few times before
|
|
// reporting the last failure.
|
|
const ATTEMPTS: u32 = 3;
|
|
let mut last_error: Box<dyn Error> = String::new().into();
|
|
for attempt in 1..=ATTEMPTS {
|
|
match download_file_checksum_once(url, checksum, algo, target_dir, view).await {
|
|
Ok(()) => return Ok(()),
|
|
Err(e) => {
|
|
log::warn!("download of '{url}' failed (attempt {attempt}/{ATTEMPTS}): {e}");
|
|
last_error = e;
|
|
tokio::time::sleep(std::time::Duration::from_millis(500 * u64::from(attempt)))
|
|
.await;
|
|
}
|
|
}
|
|
}
|
|
Err(format!("downloading '{url}' failed after {ATTEMPTS} attempts: {last_error}").into())
|
|
}
|
|
|
|
/// One download attempt of [`download_file_checksum`], verifying the
|
|
/// content length and the expected checksum
|
|
async fn download_file_checksum_once(
|
|
url: &str,
|
|
checksum: &str,
|
|
algo: crate::package_info::ChecksumAlgo,
|
|
target_dir: &Path,
|
|
view: &dyn BuildView,
|
|
) -> Result<(), Box<dyn Error>> {
|
|
// Download with the shared client (connect timeout). Large orig tarballs
|
|
// can legitimately take longer than the client's default total timeout,
|
|
// so use a generous per-request timeout for streaming downloads
|
|
let response = crate::distro_info::http_get_retried_with_timeout(
|
|
url,
|
|
Some(std::time::Duration::from_secs(30 * 60)),
|
|
)
|
|
.await
|
|
.map_err(|e| Box::new(e) as Box<dyn Error>)?;
|
|
if !response.status().is_success() {
|
|
return Err(format!("Failed to download '{}' : {}", url, response.status()).into());
|
|
}
|
|
let total_size = response.content_length();
|
|
|
|
// Target file: extract file name from URL
|
|
let filename = Path::new(url)
|
|
.file_name()
|
|
.and_then(|f| f.to_str())
|
|
.ok_or_else(|| {
|
|
format!("Could not determine a file name from URL '{url}' to download the package file")
|
|
})?;
|
|
let path = target_dir.join(filename);
|
|
let mut file = File::create(path)?;
|
|
|
|
// Download chunk by chunk to disk, while updating hasher for checksum
|
|
let mut stream = response.bytes_stream();
|
|
// Accumulate the downloaded bytes so we can compute the final digest with the
|
|
// correct algorithm once the download is complete.
|
|
let mut buffer: Vec<u8> = Vec::with_capacity(total_size.unwrap_or(0) as usize);
|
|
while let Some(item) = stream.next().await {
|
|
let chunk = item?;
|
|
file.write_all(&chunk)?;
|
|
buffer.extend_from_slice(&chunk);
|
|
|
|
if let Some(total) = total_size {
|
|
view.byte_progress("Downloading...", min(buffer.len() as u64, total), total);
|
|
}
|
|
}
|
|
|
|
// A dropped connection can end the stream early: never hand a truncated
|
|
// file to the checksum check (its mismatch message would hide the cause)
|
|
if let Some(total) = total_size
|
|
&& buffer.len() != total as usize
|
|
{
|
|
return Err(format!(
|
|
"incomplete download from '{url}': got {} of {total} bytes",
|
|
buffer.len()
|
|
)
|
|
.into());
|
|
}
|
|
|
|
// Verify checksum using the algorithm specified for this file
|
|
let calculated_checksum = algo.hex_digest(&buffer);
|
|
if calculated_checksum != checksum {
|
|
return Err(format!(
|
|
"Checksum mismatch! Expected {}, got {}",
|
|
checksum, calculated_checksum
|
|
)
|
|
.into());
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
fn setup_pristine_tar_branch(package_dir: &Path, dist: &str) -> Result<(), Box<dyn Error>> {
|
|
let repo = git2::Repository::open(package_dir)?;
|
|
|
|
// Check if local branch already exists
|
|
if repo
|
|
.find_branch("pristine-tar", git2::BranchType::Local)
|
|
.is_ok()
|
|
{
|
|
return Ok(());
|
|
}
|
|
|
|
// Find remote pristine-tar branch
|
|
let branches = repo.branches(Some(git2::BranchType::Remote))?;
|
|
for branch_result in branches {
|
|
let (branch, _) = branch_result?;
|
|
if let Some(name) = branch.name()?
|
|
&& name.ends_with(&format!("/{dist}/pristine-tar"))
|
|
{
|
|
debug!("Found remote pristine-tar branch: {}", name);
|
|
|
|
let commit = branch.get().peel_to_commit()?;
|
|
|
|
// Create local branch
|
|
let mut local_branch = repo.branch("pristine-tar", &commit, false)?;
|
|
|
|
// Set upstream
|
|
local_branch.set_upstream(Some(name))?;
|
|
|
|
debug!("Created local pristine-tar branch tracking {}", name);
|
|
return Ok(());
|
|
}
|
|
}
|
|
|
|
debug!("No remote pristine-tar branch found.");
|
|
Ok(())
|
|
}
|
|
|
|
async fn fetch_orig_tarball(
|
|
info: &PackageInfo,
|
|
cwd: Option<&Path>,
|
|
view: &dyn BuildView,
|
|
) -> Result<(), Box<dyn Error>> {
|
|
let package_dir = if let Some(path) = cwd {
|
|
path.join(&info.stanza.package)
|
|
} else {
|
|
Path::new(&info.stanza.package).to_path_buf()
|
|
};
|
|
|
|
// Upstream tarballs in the file list: the main orig tarball plus, for
|
|
// multi-orig ("3.0 (quilt)" extra-component) sources, one component
|
|
// tarball per bundled module (`*.orig-<component>.tar.<ext>`). dpkg-source
|
|
// unpacks all of them side by side, so a git pull must fetch them all.
|
|
let orig_files: Vec<_> = info
|
|
.stanza
|
|
.files
|
|
.iter()
|
|
.filter(|f| crate::build::changes::is_orig_tarball(&f.name))
|
|
.collect();
|
|
if orig_files.is_empty() {
|
|
return Err(format!(
|
|
"Could not find orig tarball in file list for package '{}'. \
|
|
Available files: {:?}",
|
|
info.stanza.package,
|
|
info.stanza
|
|
.files
|
|
.iter()
|
|
.map(|f| &f.name)
|
|
.collect::<Vec<_>>()
|
|
)
|
|
.into());
|
|
}
|
|
|
|
// 1. Try executing pristine-tar
|
|
|
|
// Setup pristine-tar branch if needed (by tracking remote branch)
|
|
let _ = setup_pristine_tar_branch(&package_dir, info.dist.as_str());
|
|
|
|
for orig_file in orig_files {
|
|
let filename = &orig_file.name;
|
|
|
|
if let Err(e) = checkout_pristine_tar(&package_dir, filename.as_str()) {
|
|
debug!(
|
|
"pristine-tar failed: {}. Falling back to archive download.",
|
|
e
|
|
);
|
|
|
|
// 2. Fallback to archive download
|
|
// We download to the parent directory of the package repo (which is standard for build tools)
|
|
// or the current directory if cwd is None (which effectively is the parent of the package dir)
|
|
let target_dir = cwd.unwrap_or_else(|| Path::new("."));
|
|
download_file_checksum(
|
|
format!("{}/{}", info.archive_url, filename).as_str(),
|
|
&orig_file.checksum,
|
|
orig_file.checksum_algo,
|
|
target_dir,
|
|
view,
|
|
)
|
|
.await?;
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
async fn fetch_dsc_file(
|
|
info: &PackageInfo,
|
|
cwd: Option<&Path>,
|
|
view: &dyn BuildView,
|
|
) -> Result<(), Box<dyn Error>> {
|
|
let target_dir = cwd.unwrap_or_else(|| Path::new("."));
|
|
|
|
// Find the dsc file in the file list
|
|
let dsc_file = info
|
|
.stanza
|
|
.files
|
|
.iter()
|
|
.find(|f| f.name.ends_with(".dsc"))
|
|
.ok_or("Could not find .dsc file in package info")?;
|
|
let filename = &dsc_file.name;
|
|
|
|
debug!("Fetching dsc file: {}", filename);
|
|
|
|
download_file_checksum(
|
|
format!("{}/{}", info.archive_url, filename).as_str(),
|
|
&dsc_file.checksum,
|
|
dsc_file.checksum_algo,
|
|
target_dir,
|
|
view,
|
|
)
|
|
.await?;
|
|
|
|
Ok(())
|
|
}
|
|
|
|
async fn fetch_archive_sources(
|
|
info: &PackageInfo,
|
|
cwd: Option<&Path>,
|
|
view: &dyn BuildView,
|
|
) -> Result<(), Box<dyn Error>> {
|
|
let package_dir = if let Some(path) = cwd {
|
|
path
|
|
} else {
|
|
&Path::new(".").to_path_buf()
|
|
};
|
|
|
|
std::fs::create_dir_all(package_dir)?;
|
|
|
|
// dpkg-source assembles a multi-orig source by unpacking the main orig
|
|
// first and the extra components over it, so component files win where
|
|
// they overlap the main tree. Process the archives in the same order,
|
|
// or the merged tree diverges from what the source package contains.
|
|
fn archive_order(name: &str) -> usize {
|
|
if name.contains(".orig-") {
|
|
1
|
|
} else if name.contains(".orig.tar.") {
|
|
0
|
|
} else if name.contains("debian.tar.") {
|
|
2
|
|
} else {
|
|
3
|
|
}
|
|
}
|
|
let mut files: Vec<&crate::package_info::FileEntry> = info.stanza.files.iter().collect();
|
|
files.sort_by_key(|file| archive_order(&file.name));
|
|
|
|
for file in &files {
|
|
let url = format!("{}/{}", info.archive_url, file.name);
|
|
download_file_checksum(&url, &file.checksum, file.checksum_algo, package_dir, view).await?;
|
|
|
|
// Extract all tar archives. Every tar of the source package matches
|
|
// (orig, debian tar, native tar, extra components); a tarball in an
|
|
// unsupported compression fails loudly in extract_archive rather
|
|
// than being left unextracted — a silently missing tree only
|
|
// surfaces as warnings from dpkg-source and broken patches later.
|
|
// Detached signatures ('.tar.gz.asc' and friends) are not tars.
|
|
let is_tar = !file.name.ends_with(".asc")
|
|
&& (file.name.contains(".tar.")
|
|
|| file.name.ends_with(".tgz")
|
|
|| file.name.ends_with(".tbz2"));
|
|
if is_tar {
|
|
let path = package_dir.join(&file.name);
|
|
let extract_dir = package_dir.join(&info.stanza.package);
|
|
|
|
let extracted = extract_archive(&path, &extract_dir, view)?;
|
|
|
|
// Special case: the debian tar does only contain 'debian'
|
|
if file.name.contains("debian.tar.") {
|
|
continue;
|
|
}
|
|
|
|
// The tree root is the extraction directory itself: an archive
|
|
// wrapped in a 'package-version/' directory (the usual upstream
|
|
// layout, and multi-orig components carry their own) has that
|
|
// wrapper unwrapped into where dpkg-source expects its content —
|
|
// the root for the main orig, '<root>/<addon>' for an extra
|
|
// component ('.orig-<addon>.tar'), replacing any pre-existing
|
|
// addon directory (dpkg-source's own multi-orig assembly). The
|
|
// wrapper is the one first path component every entry of THIS
|
|
// archive shares — recognizing it per archive rather than from
|
|
// the shape of the tree root keeps every component of a
|
|
// multi-orig package unwrapping after the first one filled the
|
|
// root.
|
|
if let Some(wrapper) = extracted.wrapper {
|
|
unwrap_into(&extract_dir, &wrapper, component_addon(&file.name))?;
|
|
}
|
|
}
|
|
|
|
// Extract and apply .diff.gz if present (old packages)
|
|
if file.name.ends_with(".diff.gz") {
|
|
let diff_gz_path = package_dir.join(&file.name);
|
|
let source_dir = package_dir.join(&info.stanza.package);
|
|
|
|
// Create the .diff file path by replacing .gz with empty string
|
|
let diff_path = diff_gz_path.with_extension("");
|
|
|
|
// Decompress the .diff.gz file directly to .diff
|
|
let input_file = File::open(&diff_gz_path)?;
|
|
let mut decoder = GzDecoder::new(input_file);
|
|
let mut output_file = File::create(&diff_path)?;
|
|
|
|
std::io::copy(&mut decoder, &mut output_file)?;
|
|
|
|
// Use relative path for the diff file (it's in the parent directory)
|
|
let relative_diff_path =
|
|
format!("../{}", diff_path.file_name().unwrap().to_string_lossy());
|
|
|
|
// Apply the patch using the patch command with relative path
|
|
let output = Command::new("patch")
|
|
.current_dir(&source_dir)
|
|
.arg("-p1")
|
|
.arg("--input")
|
|
.arg(&relative_diff_path)
|
|
.output()?;
|
|
|
|
if !output.status.success() {
|
|
// patch reports rejected hunks on stdout, not stderr
|
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
return Err(format!(
|
|
"Failed to apply patch: {}\n{}{}",
|
|
diff_path.display(),
|
|
stdout,
|
|
stderr
|
|
)
|
|
.into());
|
|
}
|
|
|
|
debug!("Successfully applied patch: {}", diff_path.display());
|
|
|
|
// Clean up the extracted .diff file
|
|
std::fs::remove_file(&diff_path)?;
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Pull a source package locally using pre-retrieved package information
|
|
///
|
|
/// This function takes a PackageInfo struct and downloads the package using the preferred method
|
|
/// (either git or direct archive download), as well as orig tarball, inside 'package' directory.
|
|
/// The source will be extracted under 'package/package'.
|
|
pub async fn pull(
|
|
package_info: &PackageInfo,
|
|
cwd: Option<&Path>,
|
|
view: &dyn BuildView,
|
|
force_archive: bool,
|
|
) -> Result<(), Box<dyn Error>> {
|
|
let package = &package_info.stanza.package;
|
|
let series = &package_info.series;
|
|
let package_dir = if let Some(path) = cwd {
|
|
path.join(package)
|
|
} else {
|
|
Path::new(package).to_path_buf()
|
|
};
|
|
|
|
if package_dir.exists() {
|
|
return Err(format!(
|
|
"Directory '{}' already exists. Remove it first, or change working directory.",
|
|
package_dir.display()
|
|
)
|
|
.into());
|
|
}
|
|
|
|
/* Fetch the package: either via git (preferred VCS) or the archive */
|
|
if let Some(ref url) = package_info.preferred_vcs
|
|
&& !force_archive
|
|
{
|
|
// We have found a preferred VCS (git repository) for the package, so
|
|
// we fetch the package from that repo.
|
|
|
|
// Depending on target series, we pick target branch; if latest series is specified,
|
|
// we target the development branch, i.e. the default branch
|
|
// Only use Ubuntu-specific branch naming if the VCS is from Launchpad
|
|
let is_launchpad_vcs = url.contains("launchpad.net");
|
|
let series_list =
|
|
crate::distro_info::get_ordered_series_name(package_info.dist.as_str()).await?;
|
|
let latest_series = series_list.first().ok_or_else(|| {
|
|
format!(
|
|
"No series information available for distribution '{}', \
|
|
cannot determine its development series to select the git branch. \
|
|
The 'distro-info' package provides this data.",
|
|
package_info.dist
|
|
)
|
|
})?;
|
|
let branch_name = if latest_series != series {
|
|
if package_info.dist == "ubuntu" && is_launchpad_vcs {
|
|
Some(format!("{}/{}", package_info.dist, series))
|
|
} else {
|
|
// Debian does not have reliable branch naming...
|
|
// Also, Ubuntu packages with salsa VCS don't have Ubuntu-specific branches
|
|
// For now, we skip that part and clone default
|
|
// TODO: Inspect remote branches and tags for matches
|
|
None
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
|
|
view.message(&format!(
|
|
"Cloning {}{}...",
|
|
url,
|
|
if let Some(b) = &branch_name {
|
|
format!(" (branch {})", b)
|
|
} else {
|
|
String::new()
|
|
}
|
|
));
|
|
|
|
clone_repo(
|
|
url.as_str(),
|
|
package,
|
|
branch_name.as_deref(),
|
|
Some(&package_dir),
|
|
view,
|
|
)?;
|
|
|
|
if !package_info.is_native() {
|
|
view.message("Fetching orig tarball...");
|
|
fetch_orig_tarball(package_info, Some(&package_dir), view).await?;
|
|
} else {
|
|
debug!("Native package, skipping orig tarball fetch.");
|
|
}
|
|
|
|
view.message("Fetching dsc file...");
|
|
fetch_dsc_file(package_info, Some(&package_dir), view).await?;
|
|
} else {
|
|
// Fallback to archive fetching
|
|
view.message("Downloading from archive...");
|
|
fetch_archive_sources(package_info, Some(&package_dir), view).await?;
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
async fn test_pull_package_end_to_end(
|
|
package: &str,
|
|
series: Option<&str>,
|
|
dist: Option<&str>,
|
|
archive: Option<bool>,
|
|
) {
|
|
// This test verifies that 'pkh pull' clones the repo and fetches the tarball.
|
|
|
|
// For determinism, we require for tests that either a distro or series is specified,
|
|
// as no distribution would mean fallback to system distro
|
|
assert!(dist.is_some() || series.is_some());
|
|
|
|
// Use a temp directory as working directory
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let cwd = temp_dir.path();
|
|
|
|
// Main 'pull' command: the one we want to test
|
|
let info = crate::package_info::lookup(
|
|
package,
|
|
None,
|
|
series,
|
|
"",
|
|
dist,
|
|
None,
|
|
None,
|
|
&crate::report::Quiet,
|
|
)
|
|
.await
|
|
.unwrap();
|
|
pull(
|
|
&info,
|
|
Some(cwd),
|
|
&crate::report::Quiet,
|
|
archive.unwrap_or(false),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
|
|
let package_dir = cwd.join(package);
|
|
assert!(package_dir.exists());
|
|
let package_source_dir = package_dir.join(package);
|
|
assert!(
|
|
package_source_dir.exists(),
|
|
"Package git repo directory not created"
|
|
);
|
|
assert!(
|
|
package_source_dir.join("debian").exists(),
|
|
"debian directory not present"
|
|
);
|
|
|
|
if package_source_dir.join(".git").exists() {
|
|
// Verify we are on the correct branch
|
|
let repo = git2::Repository::open(&package_source_dir).unwrap();
|
|
let head = repo.head().unwrap();
|
|
let name = head.name().unwrap();
|
|
|
|
// Check if the VCS is from Launchpad - only Launchpad has Ubuntu-specific branches
|
|
let is_launchpad_vcs = info
|
|
.preferred_vcs
|
|
.as_ref()
|
|
.map(|url| url.contains("launchpad.net"))
|
|
.unwrap_or(false);
|
|
|
|
if let Some(s) = series {
|
|
// The local branch should be named dist/series
|
|
// We skip debian for now as it does not have a reliable naming scheme
|
|
// Also skip Ubuntu packages with non-Launchpad VCS (e.g., salsa.debian.org)
|
|
if info.dist == "ubuntu" && is_launchpad_vcs {
|
|
assert_eq!(name, format!("refs/heads/{0}/{s}", info.dist));
|
|
}
|
|
} else {
|
|
// The local branch should be named ubuntu/devel for Ubuntu
|
|
// Debian unfortunately does not have a reliable naming scheme
|
|
// Also skip Ubuntu packages with non-Launchpad VCS
|
|
// Given that there was no series specified, and this is a test,
|
|
// we require to have a distribution specified
|
|
if dist.unwrap() == "ubuntu" && is_launchpad_vcs {
|
|
assert_eq!(name, "refs/heads/ubuntu/devel");
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check for the orig tarballs in the package dir (only for non-native
|
|
// packages): every orig listed in the stanza must be present, including
|
|
// the component tarballs of multi-orig packages (dpkg-source needs them
|
|
// all to unpack the merged upstream tree)
|
|
let mut found_dsc = false;
|
|
for entry in std::fs::read_dir(&package_dir).unwrap() {
|
|
let entry = entry.unwrap();
|
|
let name = entry.file_name().to_string_lossy().to_string();
|
|
if name.ends_with(".dsc") {
|
|
found_dsc = true;
|
|
}
|
|
}
|
|
|
|
if !info.is_native() {
|
|
for file in &info.stanza.files {
|
|
if crate::build::changes::is_orig_tarball(&file.name) {
|
|
assert!(
|
|
package_dir.join(&file.name).exists(),
|
|
"Orig tarball '{}' not found in package dir",
|
|
file.name
|
|
);
|
|
}
|
|
}
|
|
}
|
|
assert!(found_dsc, "DSC file not found in package dir");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_pull_hello_ubuntu_end_to_end() {
|
|
test_pull_package_end_to_end("hello", Some("noble"), None, None).await;
|
|
}
|
|
#[tokio::test]
|
|
async fn test_pull_hello_debian_end_to_end() {
|
|
test_pull_package_end_to_end("hello", Some("bookworm"), None, None).await;
|
|
}
|
|
|
|
/// Specific test for a package using a .diff.gz, instead of .debian and .orig
|
|
#[tokio::test]
|
|
async fn test_pull_linux_riscv_ubuntu_end_to_end() {
|
|
test_pull_package_end_to_end("linux-riscv", Some("noble"), None, Some(true)).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_pull_2048_universe_ubuntu_end_to_end() {
|
|
test_pull_package_end_to_end("2048", Some("noble"), None, None).await;
|
|
}
|
|
#[tokio::test]
|
|
async fn test_pull_1oom_contrib_debian_end_to_end() {
|
|
test_pull_package_end_to_end("1oom", Some("trixie"), None, None).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_pull_agg_svn_fallback_ok() {
|
|
test_pull_package_end_to_end("agg", Some("trixie"), None, None).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_pull_hello_debian_latest_end_to_end() {
|
|
test_pull_package_end_to_end("hello", None, Some("debian"), None).await;
|
|
}
|
|
#[tokio::test]
|
|
async fn test_pull_hello_ubuntu_latest_end_to_end() {
|
|
test_pull_package_end_to_end("hello", None, Some("ubuntu"), None).await;
|
|
}
|
|
|
|
/// Test for paraview - a package that has no Ubuntu Launchpad code,
|
|
/// only a debian salsa repo, even in Ubuntu.
|
|
/// Furthermore, paraview has a Vcs-Git value of:
|
|
/// Vcs-Git: https://salsa.debian.org/science-team/paraview.git -b debian/latest
|
|
/// Given that it is not only an url but also specifies a branch, it needs
|
|
/// special care, that this test ensures.
|
|
#[tokio::test]
|
|
async fn test_pull_paraview_ubuntu_end_to_end() {
|
|
test_pull_package_end_to_end("paraview", Some("noble"), None, None).await;
|
|
}
|
|
|
|
/// Multi-orig ("3.0 (quilt)" extra component) regression test: node-jest
|
|
/// ships its bundled modules as separate `*.orig-<component>.tar.xz`
|
|
/// tarballs next to the main orig. The git pull path must fetch every
|
|
/// component, or the later `dpkg-source -b` quilt verification fails
|
|
/// with "can't find file to patch" on the first patch touching a
|
|
/// component directory.
|
|
#[tokio::test]
|
|
async fn test_pull_node_jest_debian_end_to_end() {
|
|
test_pull_package_end_to_end("node-jest", Some("trixie"), None, None).await;
|
|
}
|
|
|
|
/// Build a minimal uncompressed ustar archive from (name, data) entries.
|
|
///
|
|
/// Raw header blocks are crafted instead of using `tar::Builder` because
|
|
/// the builder itself refuses entry names containing '..' or absolute
|
|
/// paths, which is exactly what the traversal tests need to exercise.
|
|
fn build_tar(entries: &[(&str, &[u8])]) -> Vec<u8> {
|
|
let mut out = Vec::new();
|
|
for (name, data) in entries {
|
|
push_tar_entry(&mut out, name, b'0', data, None);
|
|
}
|
|
out.extend_from_slice(&[0u8; 1024]);
|
|
out
|
|
}
|
|
|
|
/// Append one raw ustar entry: `typeflag` selects the type ('0' regular
|
|
/// file, '1' hard link, '2' symlink) and `linkname` is the stored link
|
|
/// target for the link types.
|
|
fn push_tar_entry(
|
|
out: &mut Vec<u8>,
|
|
name: &str,
|
|
typeflag: u8,
|
|
data: &[u8],
|
|
linkname: Option<&str>,
|
|
) {
|
|
let mut block = [0u8; 512];
|
|
block[..name.len()].copy_from_slice(name.as_bytes());
|
|
block[100..108].copy_from_slice(b"0000644\0");
|
|
block[124..136].copy_from_slice(format!("{:011o}\0", data.len()).as_bytes());
|
|
block[136..148].copy_from_slice(b"00000000000\0");
|
|
block[156] = typeflag;
|
|
if let Some(linkname) = linkname {
|
|
block[157..157 + linkname.len()].copy_from_slice(linkname.as_bytes());
|
|
}
|
|
block[257..263].copy_from_slice(b"ustar\0");
|
|
block[263..265].copy_from_slice(b"00");
|
|
|
|
// Checksum: sum of the header bytes with the checksum field
|
|
// (bytes 148..156) taken as spaces
|
|
let mut checksum: u32 = 0;
|
|
for (i, byte) in block.iter().enumerate() {
|
|
checksum += if (148..156).contains(&i) {
|
|
u32::from(b' ')
|
|
} else {
|
|
u32::from(*byte)
|
|
};
|
|
}
|
|
block[148..154].copy_from_slice(format!("{checksum:06o}").as_bytes());
|
|
block[154] = 0;
|
|
block[155] = b' ';
|
|
|
|
out.extend_from_slice(&block);
|
|
out.extend_from_slice(data);
|
|
let padding = (512 - (data.len() % 512)) % 512;
|
|
out.extend_from_slice(&vec![0u8; padding]);
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_tar_rejects_parent_dir_traversal() {
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let tar_path = temp_dir.path().join("malicious.orig.tar");
|
|
std::fs::write(
|
|
&tar_path,
|
|
build_tar(&[("good.txt", b"ok"), ("../evil.txt", b"pwned")]),
|
|
)
|
|
.unwrap();
|
|
|
|
let dest = temp_dir.path().join("dest");
|
|
let result = extract_tar_archive(&tar_path, &dest, &crate::report::Quiet, |f| f);
|
|
|
|
let err = match result {
|
|
Ok(_) => panic!("extraction of a traversal archive should fail"),
|
|
Err(e) => e,
|
|
};
|
|
assert!(
|
|
err.to_string().contains("../evil.txt"),
|
|
"error should name the offending entry, got: {err}"
|
|
);
|
|
// Nothing may be written outside of the destination directory
|
|
assert!(!temp_dir.path().join("evil.txt").exists());
|
|
// Legitimate entries preceding the malicious one are still extracted
|
|
assert_eq!(
|
|
std::fs::read_to_string(dest.join("good.txt")).unwrap(),
|
|
"ok"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_tar_rejects_absolute_path() {
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let tar_path = temp_dir.path().join("malicious.orig.tar");
|
|
std::fs::write(
|
|
&tar_path,
|
|
build_tar(&[
|
|
("good.txt", b"ok"),
|
|
("/pkh_test_absolute_escape.txt", b"pwned"),
|
|
]),
|
|
)
|
|
.unwrap();
|
|
|
|
let dest = temp_dir.path().join("dest");
|
|
let result = extract_tar_archive(&tar_path, &dest, &crate::report::Quiet, |f| f);
|
|
|
|
let err = match result {
|
|
Ok(_) => panic!("extraction of an absolute-path archive should fail"),
|
|
Err(e) => e,
|
|
};
|
|
assert!(
|
|
err.to_string().contains("pkh_test_absolute_escape.txt"),
|
|
"error should name the offending entry, got: {err}"
|
|
);
|
|
// Nothing may be written at the filesystem root
|
|
assert!(!Path::new("/pkh_test_absolute_escape.txt").exists());
|
|
assert_eq!(
|
|
std::fs::read_to_string(dest.join("good.txt")).unwrap(),
|
|
"ok"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_tar_archive_normal_entries() {
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let tar_path = temp_dir.path().join("normal.orig.tar");
|
|
// Raw names: 'tar::Builder' would normalize away the './' prefix
|
|
std::fs::write(
|
|
&tar_path,
|
|
build_tar(&[
|
|
("hello-1.0/file.txt", b"hello"),
|
|
("./debian/rules", b"#!/m"),
|
|
]),
|
|
)
|
|
.unwrap();
|
|
|
|
let dest = temp_dir.path().join("dest");
|
|
extract_tar_archive(&tar_path, &dest, &crate::report::Quiet, |f| f).unwrap();
|
|
|
|
assert_eq!(
|
|
std::fs::read_to_string(dest.join("hello-1.0/file.txt")).unwrap(),
|
|
"hello"
|
|
);
|
|
assert_eq!(
|
|
std::fs::read_to_string(dest.join("debian/rules")).unwrap(),
|
|
"#!/m"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_tar_hardlink_resolved_against_archive_root() {
|
|
// Regression: a hard link entry (e.g. the jq orig tarball) aborted
|
|
// the whole extraction when its target was not found where the
|
|
// extractor resolved it; tar stores link targets relative to the
|
|
// archive root
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let tar_path = temp_dir.path().join("hardlink.orig.tar");
|
|
let mut tar_bytes = Vec::new();
|
|
push_tar_entry(&mut tar_bytes, "pkg-1.0/manual.txt", b'0', b"content", None);
|
|
push_tar_entry(
|
|
&mut tar_bytes,
|
|
"pkg-1.0/docs/v1/manual.txt",
|
|
b'1',
|
|
b"",
|
|
Some("pkg-1.0/manual.txt"),
|
|
);
|
|
tar_bytes.extend_from_slice(&[0u8; 1024]);
|
|
std::fs::write(&tar_path, tar_bytes).unwrap();
|
|
|
|
let dest = temp_dir.path().join("dest");
|
|
extract_tar_archive(&tar_path, &dest, &crate::report::Quiet, |f| f).unwrap();
|
|
|
|
assert_eq!(
|
|
std::fs::read_to_string(dest.join("pkg-1.0/docs/v1/manual.txt")).unwrap(),
|
|
"content"
|
|
);
|
|
// A real hard link shares the inode with its target
|
|
use std::os::unix::fs::MetadataExt;
|
|
assert_eq!(
|
|
dest.join("pkg-1.0/manual.txt").metadata().unwrap().ino(),
|
|
dest.join("pkg-1.0/docs/v1/manual.txt")
|
|
.metadata()
|
|
.unwrap()
|
|
.ino()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_tar_skips_pax_global_header() {
|
|
// Regression: git-archive tarballs open with a pax global header
|
|
// record ('pax_global_header'): GNU tar consumes it and never
|
|
// lists or extracts it, but the tar crate yields it as an entry.
|
|
// Counting it for the wrapper detection made every git-archive
|
|
// orig look multi-rooted: the unwrap was skipped, the sources
|
|
// stayed nested one level too deep and the package's .diff.gz
|
|
// did not apply (linux-riscv).
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let tar_path = temp_dir.path().join("git.orig.tar");
|
|
let mut tar_bytes = Vec::new();
|
|
push_tar_entry(
|
|
&mut tar_bytes,
|
|
"pax_global_header",
|
|
b'g',
|
|
b"52 comment=d0e1c2d",
|
|
None,
|
|
);
|
|
push_tar_entry(&mut tar_bytes, "pkg-1.0/Makefile", b'0', b"all:", None);
|
|
tar_bytes.extend_from_slice(&[0u8; 1024]);
|
|
std::fs::write(&tar_path, tar_bytes).unwrap();
|
|
|
|
let tree = temp_dir.path().join("tree");
|
|
let extracted =
|
|
extract_tar_archive(&tar_path, &tree, &crate::report::Quiet, |f| f).unwrap();
|
|
unwrap_wrapper(&tree, &extracted.wrapper.unwrap()).unwrap();
|
|
|
|
assert!(!tree.join("pax_global_header").exists());
|
|
assert_eq!(
|
|
std::fs::read_to_string(tree.join("Makefile")).unwrap(),
|
|
"all:"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_tar_rejects_hardlink_escape() {
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let tar_path = temp_dir.path().join("malicious.orig.tar");
|
|
let mut tar_bytes = Vec::new();
|
|
push_tar_entry(&mut tar_bytes, "good.txt", b'0', b"ok", None);
|
|
push_tar_entry(&mut tar_bytes, "evil.txt", b'1', b"", Some("../evil.txt"));
|
|
tar_bytes.extend_from_slice(&[0u8; 1024]);
|
|
std::fs::write(&tar_path, tar_bytes).unwrap();
|
|
|
|
let dest = temp_dir.path().join("dest");
|
|
let result = extract_tar_archive(&tar_path, &dest, &crate::report::Quiet, |f| f);
|
|
|
|
let err = match result {
|
|
Ok(_) => panic!("extraction of an escaping hard link should fail"),
|
|
Err(e) => e,
|
|
};
|
|
assert!(
|
|
err.to_string().contains("../evil.txt"),
|
|
"error should name the offending link target, got: {err}"
|
|
);
|
|
assert!(!temp_dir.path().join("evil.txt").exists());
|
|
assert_eq!(
|
|
std::fs::read_to_string(dest.join("good.txt")).unwrap(),
|
|
"ok"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_unwrap_wrapper_without_dir_entries() {
|
|
// Regression: tarballs carrying no explicit entries for their
|
|
// directories (e.g. the 'tree' orig) used to skip the relocation
|
|
// entirely, leaving the sources nested one level too deep and
|
|
// dpkg-source rejecting the tree as 'unexpected upstream changes'
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let tar_path = temp_dir.path().join("wrapped.orig.tar");
|
|
std::fs::write(
|
|
&tar_path,
|
|
build_tar(&[
|
|
("tree-2.3.2/Makefile", b"all:"),
|
|
("tree-2.3.2/doc/tree.1", b"th"),
|
|
]),
|
|
)
|
|
.unwrap();
|
|
|
|
let tree = temp_dir.path().join("tree");
|
|
let extracted =
|
|
extract_tar_archive(&tar_path, &tree, &crate::report::Quiet, |f| f).unwrap();
|
|
unwrap_wrapper(&tree, &extracted.wrapper.unwrap()).unwrap();
|
|
|
|
assert_eq!(
|
|
std::fs::read_to_string(tree.join("Makefile")).unwrap(),
|
|
"all:"
|
|
);
|
|
assert_eq!(
|
|
std::fs::read_to_string(tree.join("doc/tree.1")).unwrap(),
|
|
"th"
|
|
);
|
|
assert!(!tree.join("tree-2.3.2").exists());
|
|
}
|
|
|
|
#[test]
|
|
fn test_unwrap_wrapper_preserves_symlinks() {
|
|
// Regression: the copy-based relocation used to flatten top-level
|
|
// symlinks into plain files (dpkg-source: 'cannot represent change
|
|
// to ...: old version is symlink')
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let tar_path = temp_dir.path().join("wrapped.orig.tar");
|
|
let mut tar_bytes = Vec::new();
|
|
push_tar_entry(&mut tar_bytes, "pkg-1.0/README.md", b'0', b"readme", None);
|
|
push_tar_entry(
|
|
&mut tar_bytes,
|
|
"pkg-1.0/README",
|
|
b'2',
|
|
b"",
|
|
Some("README.md"),
|
|
);
|
|
tar_bytes.extend_from_slice(&[0u8; 1024]);
|
|
std::fs::write(&tar_path, tar_bytes).unwrap();
|
|
|
|
let tree = temp_dir.path().join("tree");
|
|
let extracted =
|
|
extract_tar_archive(&tar_path, &tree, &crate::report::Quiet, |f| f).unwrap();
|
|
unwrap_wrapper(&tree, &extracted.wrapper.unwrap()).unwrap();
|
|
|
|
let meta = std::fs::symlink_metadata(tree.join("README")).unwrap();
|
|
assert!(meta.file_type().is_symlink());
|
|
assert_eq!(
|
|
std::fs::read_link(tree.join("README")).unwrap(),
|
|
Path::new("README.md")
|
|
);
|
|
assert_eq!(
|
|
std::fs::read_to_string(tree.join("README")).unwrap(),
|
|
"readme"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_unwrap_wrapper_leaves_flat_layout() {
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let tar_path = temp_dir.path().join("flat.orig.tar");
|
|
std::fs::write(
|
|
&tar_path,
|
|
build_tar(&[("Makefile", b"all:"), ("debian/rules", b"#!/m")]),
|
|
)
|
|
.unwrap();
|
|
|
|
let tree = temp_dir.path().join("tree");
|
|
let extracted =
|
|
extract_tar_archive(&tar_path, &tree, &crate::report::Quiet, |f| f).unwrap();
|
|
assert!(extracted.wrapper.is_none());
|
|
|
|
assert!(tree.join("Makefile").exists());
|
|
assert!(tree.join("debian/rules").exists());
|
|
}
|
|
|
|
#[test]
|
|
fn test_unwrap_wrapper_of_second_multi_orig_component() {
|
|
// Regression: multi-orig packages (e.g. dovecot) extract a second
|
|
// component into a tree root the first component already filled,
|
|
// and dpkg-source expects the component's content at
|
|
// '<root>/<addon>' ('pigeonhole' for '...orig-pigeonhole.tar.gz'),
|
|
// not the wrapper's own layout at the root. The addon directory
|
|
// used to never be created: every move into it failed with ENOENT
|
|
// and the pull aborted.
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let tree = temp_dir.path().join("tree");
|
|
|
|
let main = temp_dir.path().join("main.orig.tar");
|
|
std::fs::write(&main, build_tar(&[("dovecot-2.4.5/README", b"main")])).unwrap();
|
|
let extracted = extract_tar_archive(&main, &tree, &crate::report::Quiet, |f| f).unwrap();
|
|
unwrap_into(
|
|
&tree,
|
|
&extracted.wrapper.unwrap(),
|
|
component_addon("main.orig.tar"),
|
|
)
|
|
.unwrap();
|
|
|
|
let component = temp_dir.path().join("pkg_1.0.orig-pigeonhole.tar.gz");
|
|
std::fs::write(
|
|
&component,
|
|
build_tar(&[
|
|
("dovecot-pigeonhole-2.4.5/update-version.sh", b"sh"),
|
|
("dovecot-pigeonhole-2.4.5/pigeonhole/x", b"ph"),
|
|
]),
|
|
)
|
|
.unwrap();
|
|
let extracted =
|
|
extract_tar_archive(&component, &tree, &crate::report::Quiet, |f| f).unwrap();
|
|
unwrap_into(
|
|
&tree,
|
|
&extracted.wrapper.unwrap(),
|
|
component_addon("pkg_1.0.orig-pigeonhole.tar.gz"),
|
|
)
|
|
.unwrap();
|
|
|
|
assert_eq!(
|
|
std::fs::read_to_string(tree.join("README")).unwrap(),
|
|
"main"
|
|
);
|
|
assert_eq!(
|
|
std::fs::read_to_string(tree.join("pigeonhole/update-version.sh")).unwrap(),
|
|
"sh"
|
|
);
|
|
assert_eq!(
|
|
std::fs::read_to_string(tree.join("pigeonhole/pigeonhole/x")).unwrap(),
|
|
"ph"
|
|
);
|
|
assert!(!tree.join("update-version.sh").exists());
|
|
assert!(!tree.join("dovecot-2.4.5").exists());
|
|
assert!(!tree.join("dovecot-pigeonhole-2.4.5").exists());
|
|
}
|
|
|
|
#[test]
|
|
fn test_unwrap_component_already_prefixed_with_addon() {
|
|
// Regression: a component tarball whose entries already carry the
|
|
// addon prefix ('www/...' in 'sqlite3_3.53.4.orig-www.tar.xz')
|
|
// extracts to its end state. It used to be treated as a wrapper of
|
|
// the same name as the addon: unwrap_into cleared the target —
|
|
// destroying the component just extracted into it — and unwrapped
|
|
// the emptied directory into itself, leaving no 'www' at all.
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let tar_path = temp_dir.path().join("pkg_1.0.orig-www.tar.gz");
|
|
std::fs::write(
|
|
&tar_path,
|
|
build_tar(&[("www/README", b"docs"), ("www/img/x", b"img")]),
|
|
)
|
|
.unwrap();
|
|
|
|
let tree = temp_dir.path().join("tree");
|
|
let extracted =
|
|
extract_tar_archive(&tar_path, &tree, &crate::report::Quiet, |f| f).unwrap();
|
|
unwrap_into(
|
|
&tree,
|
|
&extracted.wrapper.unwrap(),
|
|
component_addon("pkg_1.0.orig-www.tar.gz"),
|
|
)
|
|
.unwrap();
|
|
|
|
assert_eq!(
|
|
std::fs::read_to_string(tree.join("www/README")).unwrap(),
|
|
"docs"
|
|
);
|
|
assert_eq!(
|
|
std::fs::read_to_string(tree.join("www/img/x")).unwrap(),
|
|
"img"
|
|
);
|
|
}
|
|
|
|
/// A minimal ustar archive of `pkg-1.0/{Makefile,doc/readme}`,
|
|
/// bz2-compressed.
|
|
const BZ2_TAR_WRAPPED: &[u8] = &[
|
|
66, 90, 104, 57, 49, 65, 89, 38, 83, 89, 88, 108, 110, 100, 0, 0, 154, 255, 128, 201, 128,
|
|
0, 16, 64, 3, 231, 144, 0, 3, 64, 0, 111, 174, 222, 0, 8, 40, 32, 0, 116, 18, 148, 218,
|
|
147, 200, 67, 33, 166, 100, 70, 65, 250, 83, 212, 18, 145, 1, 166, 128, 0, 0, 208, 62, 196,
|
|
104, 44, 169, 64, 3, 160, 68, 65, 10, 40, 112, 177, 128, 161, 163, 8, 68, 16, 80, 10, 79,
|
|
36, 5, 53, 192, 112, 39, 81, 227, 129, 9, 73, 68, 32, 93, 136, 243, 37, 49, 35, 19, 133,
|
|
118, 14, 245, 37, 8, 233, 250, 99, 27, 194, 182, 183, 86, 186, 163, 237, 147, 31, 53, 154,
|
|
53, 20, 231, 189, 6, 40, 100, 34, 7, 241, 119, 36, 83, 133, 9, 5, 134, 198, 230, 64,
|
|
];
|
|
|
|
#[test]
|
|
fn test_extract_tar_bz2() {
|
|
// Regression: '.tar.bz2' origs (e.g. busybox) were downloaded but
|
|
// never extracted — the dispatch only knew gz and xz — leaving a
|
|
// tree that dpkg-source packed as a mass of ignored deletions
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let tar_path = temp_dir.path().join("wrapped.orig.tar.bz2");
|
|
std::fs::write(&tar_path, BZ2_TAR_WRAPPED).unwrap();
|
|
|
|
let tree = temp_dir.path().join("tree");
|
|
let extracted = extract_tar_archive(
|
|
&tar_path,
|
|
&tree,
|
|
&crate::report::Quiet,
|
|
bzip2_rs::DecoderReader::new,
|
|
)
|
|
.unwrap();
|
|
unwrap_wrapper(&tree, &extracted.wrapper.unwrap()).unwrap();
|
|
|
|
assert_eq!(
|
|
std::fs::read_to_string(tree.join("Makefile")).unwrap(),
|
|
"all:"
|
|
);
|
|
assert_eq!(
|
|
std::fs::read_to_string(tree.join("doc/readme")).unwrap(),
|
|
"docs"
|
|
);
|
|
assert!(!tree.join("pkg-1.0").exists());
|
|
}
|
|
}
|