use std::cmp::min; use std::error::Error; use std::os::unix::fs::symlink; use std::path::Component; use std::path::Path; use std::path::PathBuf; use crate::package_info::PackageInfo; use std::process::Command; use log::debug; use regex::Regex; use crate::report::BuildView; fn clone_repo( url: &str, package: &str, branch: Option<&str>, cwd: Option<&Path>, view: &dyn BuildView, ) -> Result<(), Box> { let target_path = if let Some(path) = cwd { path.join(package) } else { Path::new(package).to_path_buf() }; let mut callbacks = git2::RemoteCallbacks::new(); // The sideband parser is only worth running when something listens if view.is_enabled() { // Download progress callbacks.transfer_progress(move |stats| { view.progress( "Receiving objects...", stats.received_objects(), stats.total_objects(), ); true }); // Remote progress: messages 'Remote: compressing objects 10% (34/340)' // Parse progress informations to display them in the view callbacks.sideband_progress(move |data| { let msg = String::from_utf8_lossy(data); let re = Regex::new(r"(.*):[ ]*([0-9]*)% \(([0-9]*)/([0-9]*)\)").unwrap(); if let Some(caps) = re.captures(msg.trim()) { let msg = caps.get(1).map_or("", |m| m.as_str()).to_string(); let objects = caps .get(3) .map_or("", |m| m.as_str()) .to_string() .parse::() .unwrap_or(0); let total = caps .get(4) .map_or("", |m| m.as_str()) .to_string() .parse::() .unwrap_or(0); view.progress(msg.as_str(), objects, total); } true }); } let mut fetch_options = git2::FetchOptions::new(); fetch_options.remote_callbacks(callbacks); let mut builder = git2::build::RepoBuilder::new(); builder.fetch_options(fetch_options); if let Some(b) = branch { builder.branch(b); } match builder.clone(url, &target_path) { Ok(_repo) => Ok(()), Err(e) => Err(format!("Failed to clone: {}", e).into()), } } use std::fs::File; use std::io::Write; use flate2::read::GzDecoder; use futures_util::StreamExt; use tar::Archive; use xz2::read::XzDecoder; fn copy_dir_all(src: &Path, dst: &Path) -> Result<(), Box> { if !dst.exists() { std::fs::create_dir_all(dst)?; } for entry in std::fs::read_dir(src)? { let entry = entry?; let src_path = entry.path(); let dst_path = dst.join(entry.file_name()); // Reproduce symlinks as symlinks rather than following them, so that // dangling/absolute symlinks do not abort the copy. if std::fs::symlink_metadata(&src_path)? .file_type() .is_symlink() { let target = std::fs::read_link(&src_path)?; let _ = std::fs::remove_file(&dst_path); symlink(&target, &dst_path)?; } else if src_path.is_dir() { copy_dir_all(&src_path, &dst_path)?; } else { // Copy the file, preserving the source modification and access // times. This is important for autotools/gnulib-based packages // (e.g. 'hello' from Debian sid) that ship pre-generated files // alongside their prerequisites: if the copy resets the mtime to // "now", the prerequisites appear as new as the generated targets // and `make` tries to regenerate them using tools (like gperf) // that are not declared build-dependencies. std::fs::copy(&src_path, &dst_path)?; copy_file_times(&src_path, &dst_path)?; } } Ok(()) } /// Restore the modification and access times of `dest` to match `src`. /// /// `std::fs::copy` resets the destination mtime to "now", which breaks /// timestamp-based build systems (autotools/gnulib) that ship pre-generated /// files alongside their prerequisites. Restoring the original timestamps /// prevents `make` from needlessly regenerating those files with tools that /// may not be installed (e.g. `gperf`). fn copy_file_times(src: &Path, dest: &Path) -> Result<(), Box> { let metadata = std::fs::metadata(src)?; let mut times = std::fs::FileTimes::new(); let mut have_times = false; if let Ok(mtime) = metadata.modified() { times = times.set_modified(mtime); have_times = true; } if let Ok(atime) = metadata.accessed() { times = times.set_accessed(atime); have_times = true; } if have_times && let Ok(dest_file) = std::fs::File::open(dest) { let _ = dest_file.set_times(times); } Ok(()) } /// The addon name of an extra upstream component tarball ('pigeonhole' for /// 'pkg_1.0.orig-pigeonhole.tar.gz'): dpkg-source unpacks it into /// '/'. None for the main orig and every non-component file. fn component_addon(name: &str) -> Option<&str> { let rest = name.split_once(".orig-")?.1; let addon = rest.split(".tar.").next()?; if addon.is_empty() { None } else { Some(addon) } } /// Move the contents of an archive's wrapper directory into `dir`, or into /// `dir/` for a multi-orig component, replacing any pre-existing /// addon directory. fn unwrap_into(dir: &Path, wrapper: &Path, addon: Option<&str>) -> Result<(), Box> { match addon { // dpkg-source unpacks a component into '/': a tarball // whose entries already carry that prefix ('www/...' in // 'sqlite3_3.53.4.orig-www.tar.xz') extracts to its end state, and // unwrapping it would clear the target and move the cleared // directory into itself. Some(addon) if wrapper.file_name().is_some_and(|name| name == addon) => Ok(()), Some(addon) => { let target = dir.join(addon); if target.is_dir() { std::fs::remove_dir_all(&target)?; } std::fs::create_dir_all(&target)?; unwrap_wrapper(&target, wrapper) } None => unwrap_wrapper(dir, wrapper), } } /// Move the contents of an archive's wrapper directory up into `dir` itself. /// /// Upstream tarballs are usually wrapped in a 'package-version/' directory /// while the tree root is `dir`. The wrapper's entries are moved up one /// level by rename — symlinks and timestamps survive a move, while a copy /// flattens links and resets times — and the emptied wrapper is removed. /// A name already present at the root (a multi-orig component laid files /// there before) is merged over rather than clobbered. fn unwrap_wrapper(dir: &Path, wrapper: &Path) -> Result<(), Box> { debug!("Unwrapping {} into {}", wrapper.display(), dir.display()); for entry in std::fs::read_dir(wrapper)? { let entry = entry?; let target = dir.join(entry.file_name()); if !target.exists() { std::fs::rename(entry.path(), &target)?; } else { // The tree root already carries that name (a multi-orig archive // laid files there before): merge over it instead of clobbering if entry.path().is_dir() { std::fs::create_dir_all(&target)?; copy_dir_all(&entry.path(), &target)?; std::fs::remove_dir_all(entry.path())?; } else { std::fs::copy(entry.path(), &target)?; copy_file_times(&entry.path(), &target)?; std::fs::remove_file(entry.path())?; } } } std::fs::remove_dir(wrapper)?; Ok(()) } /// Result of extracting an archive struct ExtractedArchive { /// The single first path component every entry of the archive shares: /// its 'package-version/' wrapper, to unwrap by the caller. `None` when /// the archive is already laid out in place (several first components, /// root-level files, an empty archive). wrapper: Option, } /// Helper function to extract tar archive with progress tracking fn extract_tar_archive( file_path: &Path, dest: &Path, view: &dyn BuildView, decoder_factory: F, ) -> Result> where D: std::io::Read, F: Fn(File) -> D, { let file = File::open(file_path)?; let decoder = decoder_factory(file); let mut archive = Archive::new(decoder); // Get total number of entries for progress tracking let total_entries = archive.entries()?.count(); let mut current_entry = 0; // Reset the archive to read entries again let file = File::open(file_path)?; let decoder = decoder_factory(file); let mut archive = Archive::new(decoder); // The archive's wrapper: the one first path component every entry // shares. Tracked from the entries themselves — tarballs are not // required to carry entries for their directories, and a missing entry // used to leave the sources nested one level too deep. let mut root: Option = None; let mut multiple_roots = false; for entry in archive.entries()? { let mut entry = entry?; // A pax global header record ('pax_global_header', the opening // entry of git-archive tarballs) is metadata about the archive, // not content: GNU tar consumes it and never lists or extracts // it, but the tar crate yields it as an entry. It must not be // extracted (unpack would no-op) and must not count for the // wrapper detection below — it used to make every git-archive // orig look multi-rooted, skipping the unwrap and leaving the // sources nested one level too deep. if entry.header().entry_type().is_pax_global_extensions() { continue; } let path = entry.path()?.to_path_buf(); // Archives built with './'-prefixed entries (common in third-party // repositories) target the destination directory itself, with no // 'package-version/' wrapper; strip the prefix so entries land there // too, and skip their root entry let relative = path.strip_prefix("./").unwrap_or(&path).to_path_buf(); if relative.as_os_str().is_empty() { continue; } let first = relative.components().next(); if let Some(Component::Normal(name)) = first { match root.take() { None => root = Some(name.to_os_string()), Some(shared) if shared.as_os_str() == name => root = Some(shared), Some(_) => multiple_roots = true, } } // Security: `Entry::unpack` performs no path sanitization, so extract // the entry only if its path is confined to the destination directory. // An absolute path or a path containing '..' would let a malicious (or // deeply malformed) tarball write files anywhere outside 'dest' (path // traversal). Refuse such entries with an error rather than skipping // them silently, so the problem is not hidden. let escapes_dest = relative.components().any(|component| { matches!( component, Component::Prefix(_) | Component::RootDir | Component::ParentDir ) }); if escapes_dest { return Err(format!( "Refusing to extract '{}': archive entry path is absolute or \ contains '..' and would escape the destination directory '{}'", relative.display(), dest.display() ) .into()); } let dest_path = dest.join(&relative); // Create parent directories if needed if let Some(parent) = dest_path.parent() { std::fs::create_dir_all(parent)?; } // A hard link shares its content with an earlier entry of the // archive: tar stores the target path relative to the archive root, // where the shared entry has already been extracted. Resolving it // any other way (as `Entry::unpack` does) misses the target and // aborts the whole extraction. if entry.header().entry_type().is_hard_link() { let link_target = entry.link_name()?.ok_or_else(|| { format!( "archive entry '{}' is a hard link without a link target", relative.display() ) })?; // Absolute link targets land under the destination root, like // GNU tar treats them; anything else escaping 'dest' is refused let link_relative = link_target.strip_prefix("/").unwrap_or(&link_target); let escapes_dest = link_relative.components().any(|component| { matches!( component, Component::Prefix(_) | Component::RootDir | Component::ParentDir ) }); if escapes_dest { return Err(format!( "Refusing to extract '{}': hard link target '{}' is \ absolute or contains '..' and would escape the \ destination directory '{}'", relative.display(), link_relative.display(), dest.display() ) .into()); } let link_path = dest.join(link_relative); std::fs::hard_link(&link_path, &dest_path).map_err(|e| { format!( "hard linking archive entry '{}' to its target '{}' \ failed: {e}", relative.display(), link_relative.display() ) })?; } else { // Extract the entry entry.unpack(&dest_path)?; } current_entry += 1; // Report progress view.progress("Extracting...", current_entry, total_entries); } let wrapper = if multiple_roots { None } else { root.map(|r| dest.join(r)) }; Ok(ExtractedArchive { wrapper }) } fn extract_archive( path: &Path, dest: &Path, view: &dyn BuildView, ) -> Result> { let filename = path.file_name().unwrap().to_string_lossy(); if filename.ends_with(".tar.gz") || filename.ends_with(".tgz") { extract_tar_archive(path, dest, view, GzDecoder::new) } else if filename.ends_with(".tar.xz") || filename.ends_with(".txz") { extract_tar_archive(path, dest, view, XzDecoder::new) } else if filename.ends_with(".tar.bz2") || filename.ends_with(".tbz2") { extract_tar_archive(path, dest, view, bzip2_rs::DecoderReader::new) } else { Err(format!("Unsupported archive format: {}", filename).into()) } } fn checkout_pristine_tar(package_dir: &Path, filename: &str) -> Result<(), Box> { let output = Command::new("pristine-tar") .current_dir(package_dir) .args(["checkout", format!("../{filename}").as_str()]) .output() .map_err(|e| { format!( "Failed to run 'pristine-tar' to check out '{filename}': {}. \ Is 'pristine-tar' installed? It is required to reconstruct \ the upstream orig tarball from a git repository.", e ) })?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(format!( "pristine-tar checkout of '{filename}' failed with status: {}.{}", output.status, if stderr.trim().is_empty() { String::new() } else { format!("\npristine-tar output:\n{}", stderr.trim()) } ) .into()); } Ok(()) } async fn download_file_checksum( url: &str, checksum: &str, algo: crate::package_info::ChecksumAlgo, target_dir: &Path, view: &dyn BuildView, ) -> Result<(), Box> { // Archive mirrors and CDNs are busy enough that single attempts fail // spuriously (dropped connections, truncated bodies, index generations // momentarily out of sync): retry the whole download a few times before // reporting the last failure. const ATTEMPTS: u32 = 3; let mut last_error: Box = String::new().into(); for attempt in 1..=ATTEMPTS { match download_file_checksum_once(url, checksum, algo, target_dir, view).await { Ok(()) => return Ok(()), Err(e) => { log::warn!("download of '{url}' failed (attempt {attempt}/{ATTEMPTS}): {e}"); last_error = e; tokio::time::sleep(std::time::Duration::from_millis(500 * u64::from(attempt))) .await; } } } Err(format!("downloading '{url}' failed after {ATTEMPTS} attempts: {last_error}").into()) } /// One download attempt of [`download_file_checksum`], verifying the /// content length and the expected checksum async fn download_file_checksum_once( url: &str, checksum: &str, algo: crate::package_info::ChecksumAlgo, target_dir: &Path, view: &dyn BuildView, ) -> Result<(), Box> { // Download with the shared client (connect timeout). Large orig tarballs // can legitimately take longer than the client's default total timeout, // so use a generous per-request timeout for streaming downloads let response = crate::distro_info::http_get_retried_with_timeout( url, Some(std::time::Duration::from_secs(30 * 60)), ) .await .map_err(|e| Box::new(e) as Box)?; if !response.status().is_success() { return Err(format!("Failed to download '{}' : {}", url, response.status()).into()); } let total_size = response.content_length(); // Target file: extract file name from URL let filename = Path::new(url) .file_name() .and_then(|f| f.to_str()) .ok_or_else(|| { format!("Could not determine a file name from URL '{url}' to download the package file") })?; let path = target_dir.join(filename); let mut file = File::create(path)?; // Download chunk by chunk to disk, while updating hasher for checksum let mut stream = response.bytes_stream(); // Accumulate the downloaded bytes so we can compute the final digest with the // correct algorithm once the download is complete. let mut buffer: Vec = Vec::with_capacity(total_size.unwrap_or(0) as usize); while let Some(item) = stream.next().await { let chunk = item?; file.write_all(&chunk)?; buffer.extend_from_slice(&chunk); if let Some(total) = total_size { view.byte_progress("Downloading...", min(buffer.len() as u64, total), total); } } // A dropped connection can end the stream early: never hand a truncated // file to the checksum check (its mismatch message would hide the cause) if let Some(total) = total_size && buffer.len() != total as usize { return Err(format!( "incomplete download from '{url}': got {} of {total} bytes", buffer.len() ) .into()); } // Verify checksum using the algorithm specified for this file let calculated_checksum = algo.hex_digest(&buffer); if calculated_checksum != checksum { return Err(format!( "Checksum mismatch! Expected {}, got {}", checksum, calculated_checksum ) .into()); } Ok(()) } fn setup_pristine_tar_branch(package_dir: &Path, dist: &str) -> Result<(), Box> { let repo = git2::Repository::open(package_dir)?; // Check if local branch already exists if repo .find_branch("pristine-tar", git2::BranchType::Local) .is_ok() { return Ok(()); } // Find remote pristine-tar branch let branches = repo.branches(Some(git2::BranchType::Remote))?; for branch_result in branches { let (branch, _) = branch_result?; if let Some(name) = branch.name()? && name.ends_with(&format!("/{dist}/pristine-tar")) { debug!("Found remote pristine-tar branch: {}", name); let commit = branch.get().peel_to_commit()?; // Create local branch let mut local_branch = repo.branch("pristine-tar", &commit, false)?; // Set upstream local_branch.set_upstream(Some(name))?; debug!("Created local pristine-tar branch tracking {}", name); return Ok(()); } } debug!("No remote pristine-tar branch found."); Ok(()) } async fn fetch_orig_tarball( info: &PackageInfo, cwd: Option<&Path>, view: &dyn BuildView, ) -> Result<(), Box> { let package_dir = if let Some(path) = cwd { path.join(&info.stanza.package) } else { Path::new(&info.stanza.package).to_path_buf() }; // Upstream tarballs in the file list: the main orig tarball plus, for // multi-orig ("3.0 (quilt)" extra-component) sources, one component // tarball per bundled module (`*.orig-.tar.`). dpkg-source // unpacks all of them side by side, so a git pull must fetch them all. let orig_files: Vec<_> = info .stanza .files .iter() .filter(|f| crate::build::changes::is_orig_tarball(&f.name)) .collect(); if orig_files.is_empty() { return Err(format!( "Could not find orig tarball in file list for package '{}'. \ Available files: {:?}", info.stanza.package, info.stanza .files .iter() .map(|f| &f.name) .collect::>() ) .into()); } // 1. Try executing pristine-tar // Setup pristine-tar branch if needed (by tracking remote branch) let _ = setup_pristine_tar_branch(&package_dir, info.dist.as_str()); for orig_file in orig_files { let filename = &orig_file.name; if let Err(e) = checkout_pristine_tar(&package_dir, filename.as_str()) { debug!( "pristine-tar failed: {}. Falling back to archive download.", e ); // 2. Fallback to archive download // We download to the parent directory of the package repo (which is standard for build tools) // or the current directory if cwd is None (which effectively is the parent of the package dir) let target_dir = cwd.unwrap_or_else(|| Path::new(".")); download_file_checksum( format!("{}/{}", info.archive_url, filename).as_str(), &orig_file.checksum, orig_file.checksum_algo, target_dir, view, ) .await?; } } Ok(()) } async fn fetch_dsc_file( info: &PackageInfo, cwd: Option<&Path>, view: &dyn BuildView, ) -> Result<(), Box> { let target_dir = cwd.unwrap_or_else(|| Path::new(".")); // Find the dsc file in the file list let dsc_file = info .stanza .files .iter() .find(|f| f.name.ends_with(".dsc")) .ok_or("Could not find .dsc file in package info")?; let filename = &dsc_file.name; debug!("Fetching dsc file: {}", filename); download_file_checksum( format!("{}/{}", info.archive_url, filename).as_str(), &dsc_file.checksum, dsc_file.checksum_algo, target_dir, view, ) .await?; Ok(()) } async fn fetch_archive_sources( info: &PackageInfo, cwd: Option<&Path>, view: &dyn BuildView, ) -> Result<(), Box> { let package_dir = if let Some(path) = cwd { path } else { &Path::new(".").to_path_buf() }; std::fs::create_dir_all(package_dir)?; // dpkg-source assembles a multi-orig source by unpacking the main orig // first and the extra components over it, so component files win where // they overlap the main tree. Process the archives in the same order, // or the merged tree diverges from what the source package contains. fn archive_order(name: &str) -> usize { if name.contains(".orig-") { 1 } else if name.contains(".orig.tar.") { 0 } else if name.contains("debian.tar.") { 2 } else { 3 } } let mut files: Vec<&crate::package_info::FileEntry> = info.stanza.files.iter().collect(); files.sort_by_key(|file| archive_order(&file.name)); for file in &files { let url = format!("{}/{}", info.archive_url, file.name); download_file_checksum(&url, &file.checksum, file.checksum_algo, package_dir, view).await?; // Extract all tar archives. Every tar of the source package matches // (orig, debian tar, native tar, extra components); a tarball in an // unsupported compression fails loudly in extract_archive rather // than being left unextracted — a silently missing tree only // surfaces as warnings from dpkg-source and broken patches later. // Detached signatures ('.tar.gz.asc' and friends) are not tars. let is_tar = !file.name.ends_with(".asc") && (file.name.contains(".tar.") || file.name.ends_with(".tgz") || file.name.ends_with(".tbz2")); if is_tar { let path = package_dir.join(&file.name); let extract_dir = package_dir.join(&info.stanza.package); let extracted = extract_archive(&path, &extract_dir, view)?; // Special case: the debian tar does only contain 'debian' if file.name.contains("debian.tar.") { continue; } // The tree root is the extraction directory itself: an archive // wrapped in a 'package-version/' directory (the usual upstream // layout, and multi-orig components carry their own) has that // wrapper unwrapped into where dpkg-source expects its content — // the root for the main orig, '/' for an extra // component ('.orig-.tar'), replacing any pre-existing // addon directory (dpkg-source's own multi-orig assembly). The // wrapper is the one first path component every entry of THIS // archive shares — recognizing it per archive rather than from // the shape of the tree root keeps every component of a // multi-orig package unwrapping after the first one filled the // root. if let Some(wrapper) = extracted.wrapper { unwrap_into(&extract_dir, &wrapper, component_addon(&file.name))?; } } // Extract and apply .diff.gz if present (old packages) if file.name.ends_with(".diff.gz") { let diff_gz_path = package_dir.join(&file.name); let source_dir = package_dir.join(&info.stanza.package); // Create the .diff file path by replacing .gz with empty string let diff_path = diff_gz_path.with_extension(""); // Decompress the .diff.gz file directly to .diff let input_file = File::open(&diff_gz_path)?; let mut decoder = GzDecoder::new(input_file); let mut output_file = File::create(&diff_path)?; std::io::copy(&mut decoder, &mut output_file)?; // Use relative path for the diff file (it's in the parent directory) let relative_diff_path = format!("../{}", diff_path.file_name().unwrap().to_string_lossy()); // Apply the patch using the patch command with relative path let output = Command::new("patch") .current_dir(&source_dir) .arg("-p1") .arg("--input") .arg(&relative_diff_path) .output()?; if !output.status.success() { // patch reports rejected hunks on stdout, not stderr let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); return Err(format!( "Failed to apply patch: {}\n{}{}", diff_path.display(), stdout, stderr ) .into()); } debug!("Successfully applied patch: {}", diff_path.display()); // Clean up the extracted .diff file std::fs::remove_file(&diff_path)?; } } Ok(()) } /// Pull a source package locally using pre-retrieved package information /// /// This function takes a PackageInfo struct and downloads the package using the preferred method /// (either git or direct archive download), as well as orig tarball, inside 'package' directory. /// The source will be extracted under 'package/package'. pub async fn pull( package_info: &PackageInfo, cwd: Option<&Path>, view: &dyn BuildView, force_archive: bool, ) -> Result<(), Box> { let package = &package_info.stanza.package; let series = &package_info.series; let package_dir = if let Some(path) = cwd { path.join(package) } else { Path::new(package).to_path_buf() }; if package_dir.exists() { return Err(format!( "Directory '{}' already exists. Remove it first, or change working directory.", package_dir.display() ) .into()); } /* Fetch the package: either via git (preferred VCS) or the archive */ if let Some(ref url) = package_info.preferred_vcs && !force_archive { // We have found a preferred VCS (git repository) for the package, so // we fetch the package from that repo. // Depending on target series, we pick target branch; if latest series is specified, // we target the development branch, i.e. the default branch // Only use Ubuntu-specific branch naming if the VCS is from Launchpad let is_launchpad_vcs = url.contains("launchpad.net"); let series_list = crate::distro_info::get_ordered_series_name(package_info.dist.as_str()).await?; let latest_series = series_list.first().ok_or_else(|| { format!( "No series information available for distribution '{}', \ cannot determine its development series to select the git branch. \ The 'distro-info' package provides this data.", package_info.dist ) })?; let branch_name = if latest_series != series { if package_info.dist == "ubuntu" && is_launchpad_vcs { Some(format!("{}/{}", package_info.dist, series)) } else { // Debian does not have reliable branch naming... // Also, Ubuntu packages with salsa VCS don't have Ubuntu-specific branches // For now, we skip that part and clone default // TODO: Inspect remote branches and tags for matches None } } else { None }; view.message(&format!( "Cloning {}{}...", url, if let Some(b) = &branch_name { format!(" (branch {})", b) } else { String::new() } )); clone_repo( url.as_str(), package, branch_name.as_deref(), Some(&package_dir), view, )?; if !package_info.is_native() { view.message("Fetching orig tarball..."); fetch_orig_tarball(package_info, Some(&package_dir), view).await?; } else { debug!("Native package, skipping orig tarball fetch."); } view.message("Fetching dsc file..."); fetch_dsc_file(package_info, Some(&package_dir), view).await?; } else { // Fallback to archive fetching view.message("Downloading from archive..."); fetch_archive_sources(package_info, Some(&package_dir), view).await?; } Ok(()) } #[cfg(test)] mod tests { use super::*; async fn test_pull_package_end_to_end( package: &str, series: Option<&str>, dist: Option<&str>, archive: Option, ) { // This test verifies that 'pkh pull' clones the repo and fetches the tarball. // For determinism, we require for tests that either a distro or series is specified, // as no distribution would mean fallback to system distro assert!(dist.is_some() || series.is_some()); // Use a temp directory as working directory let temp_dir = tempfile::tempdir().unwrap(); let cwd = temp_dir.path(); // Main 'pull' command: the one we want to test let info = crate::package_info::lookup( package, None, series, "", dist, None, None, &crate::report::Quiet, ) .await .unwrap(); pull( &info, Some(cwd), &crate::report::Quiet, archive.unwrap_or(false), ) .await .unwrap(); let package_dir = cwd.join(package); assert!(package_dir.exists()); let package_source_dir = package_dir.join(package); assert!( package_source_dir.exists(), "Package git repo directory not created" ); assert!( package_source_dir.join("debian").exists(), "debian directory not present" ); if package_source_dir.join(".git").exists() { // Verify we are on the correct branch let repo = git2::Repository::open(&package_source_dir).unwrap(); let head = repo.head().unwrap(); let name = head.name().unwrap(); // Check if the VCS is from Launchpad - only Launchpad has Ubuntu-specific branches let is_launchpad_vcs = info .preferred_vcs .as_ref() .map(|url| url.contains("launchpad.net")) .unwrap_or(false); if let Some(s) = series { // The local branch should be named dist/series // We skip debian for now as it does not have a reliable naming scheme // Also skip Ubuntu packages with non-Launchpad VCS (e.g., salsa.debian.org) if info.dist == "ubuntu" && is_launchpad_vcs { assert_eq!(name, format!("refs/heads/{0}/{s}", info.dist)); } } else { // The local branch should be named ubuntu/devel for Ubuntu // Debian unfortunately does not have a reliable naming scheme // Also skip Ubuntu packages with non-Launchpad VCS // Given that there was no series specified, and this is a test, // we require to have a distribution specified if dist.unwrap() == "ubuntu" && is_launchpad_vcs { assert_eq!(name, "refs/heads/ubuntu/devel"); } } } // Check for the orig tarballs in the package dir (only for non-native // packages): every orig listed in the stanza must be present, including // the component tarballs of multi-orig packages (dpkg-source needs them // all to unpack the merged upstream tree) let mut found_dsc = false; for entry in std::fs::read_dir(&package_dir).unwrap() { let entry = entry.unwrap(); let name = entry.file_name().to_string_lossy().to_string(); if name.ends_with(".dsc") { found_dsc = true; } } if !info.is_native() { for file in &info.stanza.files { if crate::build::changes::is_orig_tarball(&file.name) { assert!( package_dir.join(&file.name).exists(), "Orig tarball '{}' not found in package dir", file.name ); } } } assert!(found_dsc, "DSC file not found in package dir"); } #[tokio::test] async fn test_pull_hello_ubuntu_end_to_end() { test_pull_package_end_to_end("hello", Some("noble"), None, None).await; } #[tokio::test] async fn test_pull_hello_debian_end_to_end() { test_pull_package_end_to_end("hello", Some("bookworm"), None, None).await; } /// Specific test for a package using a .diff.gz, instead of .debian and .orig #[tokio::test] async fn test_pull_linux_riscv_ubuntu_end_to_end() { test_pull_package_end_to_end("linux-riscv", Some("noble"), None, Some(true)).await; } #[tokio::test] async fn test_pull_2048_universe_ubuntu_end_to_end() { test_pull_package_end_to_end("2048", Some("noble"), None, None).await; } #[tokio::test] async fn test_pull_1oom_contrib_debian_end_to_end() { test_pull_package_end_to_end("1oom", Some("trixie"), None, None).await; } #[tokio::test] async fn test_pull_agg_svn_fallback_ok() { test_pull_package_end_to_end("agg", Some("trixie"), None, None).await; } #[tokio::test] async fn test_pull_hello_debian_latest_end_to_end() { test_pull_package_end_to_end("hello", None, Some("debian"), None).await; } #[tokio::test] async fn test_pull_hello_ubuntu_latest_end_to_end() { test_pull_package_end_to_end("hello", None, Some("ubuntu"), None).await; } /// Test for paraview - a package that has no Ubuntu Launchpad code, /// only a debian salsa repo, even in Ubuntu. /// Furthermore, paraview has a Vcs-Git value of: /// Vcs-Git: https://salsa.debian.org/science-team/paraview.git -b debian/latest /// Given that it is not only an url but also specifies a branch, it needs /// special care, that this test ensures. #[tokio::test] async fn test_pull_paraview_ubuntu_end_to_end() { test_pull_package_end_to_end("paraview", Some("noble"), None, None).await; } /// Multi-orig ("3.0 (quilt)" extra component) regression test: node-jest /// ships its bundled modules as separate `*.orig-.tar.xz` /// tarballs next to the main orig. The git pull path must fetch every /// component, or the later `dpkg-source -b` quilt verification fails /// with "can't find file to patch" on the first patch touching a /// component directory. #[tokio::test] async fn test_pull_node_jest_debian_end_to_end() { test_pull_package_end_to_end("node-jest", Some("trixie"), None, None).await; } /// Build a minimal uncompressed ustar archive from (name, data) entries. /// /// Raw header blocks are crafted instead of using `tar::Builder` because /// the builder itself refuses entry names containing '..' or absolute /// paths, which is exactly what the traversal tests need to exercise. fn build_tar(entries: &[(&str, &[u8])]) -> Vec { let mut out = Vec::new(); for (name, data) in entries { push_tar_entry(&mut out, name, b'0', data, None); } out.extend_from_slice(&[0u8; 1024]); out } /// Append one raw ustar entry: `typeflag` selects the type ('0' regular /// file, '1' hard link, '2' symlink) and `linkname` is the stored link /// target for the link types. fn push_tar_entry( out: &mut Vec, name: &str, typeflag: u8, data: &[u8], linkname: Option<&str>, ) { let mut block = [0u8; 512]; block[..name.len()].copy_from_slice(name.as_bytes()); block[100..108].copy_from_slice(b"0000644\0"); block[124..136].copy_from_slice(format!("{:011o}\0", data.len()).as_bytes()); block[136..148].copy_from_slice(b"00000000000\0"); block[156] = typeflag; if let Some(linkname) = linkname { block[157..157 + linkname.len()].copy_from_slice(linkname.as_bytes()); } block[257..263].copy_from_slice(b"ustar\0"); block[263..265].copy_from_slice(b"00"); // Checksum: sum of the header bytes with the checksum field // (bytes 148..156) taken as spaces let mut checksum: u32 = 0; for (i, byte) in block.iter().enumerate() { checksum += if (148..156).contains(&i) { u32::from(b' ') } else { u32::from(*byte) }; } block[148..154].copy_from_slice(format!("{checksum:06o}").as_bytes()); block[154] = 0; block[155] = b' '; out.extend_from_slice(&block); out.extend_from_slice(data); let padding = (512 - (data.len() % 512)) % 512; out.extend_from_slice(&vec![0u8; padding]); } #[test] fn test_extract_tar_rejects_parent_dir_traversal() { let temp_dir = tempfile::tempdir().unwrap(); let tar_path = temp_dir.path().join("malicious.orig.tar"); std::fs::write( &tar_path, build_tar(&[("good.txt", b"ok"), ("../evil.txt", b"pwned")]), ) .unwrap(); let dest = temp_dir.path().join("dest"); let result = extract_tar_archive(&tar_path, &dest, &crate::report::Quiet, |f| f); let err = match result { Ok(_) => panic!("extraction of a traversal archive should fail"), Err(e) => e, }; assert!( err.to_string().contains("../evil.txt"), "error should name the offending entry, got: {err}" ); // Nothing may be written outside of the destination directory assert!(!temp_dir.path().join("evil.txt").exists()); // Legitimate entries preceding the malicious one are still extracted assert_eq!( std::fs::read_to_string(dest.join("good.txt")).unwrap(), "ok" ); } #[test] fn test_extract_tar_rejects_absolute_path() { let temp_dir = tempfile::tempdir().unwrap(); let tar_path = temp_dir.path().join("malicious.orig.tar"); std::fs::write( &tar_path, build_tar(&[ ("good.txt", b"ok"), ("/pkh_test_absolute_escape.txt", b"pwned"), ]), ) .unwrap(); let dest = temp_dir.path().join("dest"); let result = extract_tar_archive(&tar_path, &dest, &crate::report::Quiet, |f| f); let err = match result { Ok(_) => panic!("extraction of an absolute-path archive should fail"), Err(e) => e, }; assert!( err.to_string().contains("pkh_test_absolute_escape.txt"), "error should name the offending entry, got: {err}" ); // Nothing may be written at the filesystem root assert!(!Path::new("/pkh_test_absolute_escape.txt").exists()); assert_eq!( std::fs::read_to_string(dest.join("good.txt")).unwrap(), "ok" ); } #[test] fn test_extract_tar_archive_normal_entries() { let temp_dir = tempfile::tempdir().unwrap(); let tar_path = temp_dir.path().join("normal.orig.tar"); // Raw names: 'tar::Builder' would normalize away the './' prefix std::fs::write( &tar_path, build_tar(&[ ("hello-1.0/file.txt", b"hello"), ("./debian/rules", b"#!/m"), ]), ) .unwrap(); let dest = temp_dir.path().join("dest"); extract_tar_archive(&tar_path, &dest, &crate::report::Quiet, |f| f).unwrap(); assert_eq!( std::fs::read_to_string(dest.join("hello-1.0/file.txt")).unwrap(), "hello" ); assert_eq!( std::fs::read_to_string(dest.join("debian/rules")).unwrap(), "#!/m" ); } #[test] fn test_extract_tar_hardlink_resolved_against_archive_root() { // Regression: a hard link entry (e.g. the jq orig tarball) aborted // the whole extraction when its target was not found where the // extractor resolved it; tar stores link targets relative to the // archive root let temp_dir = tempfile::tempdir().unwrap(); let tar_path = temp_dir.path().join("hardlink.orig.tar"); let mut tar_bytes = Vec::new(); push_tar_entry(&mut tar_bytes, "pkg-1.0/manual.txt", b'0', b"content", None); push_tar_entry( &mut tar_bytes, "pkg-1.0/docs/v1/manual.txt", b'1', b"", Some("pkg-1.0/manual.txt"), ); tar_bytes.extend_from_slice(&[0u8; 1024]); std::fs::write(&tar_path, tar_bytes).unwrap(); let dest = temp_dir.path().join("dest"); extract_tar_archive(&tar_path, &dest, &crate::report::Quiet, |f| f).unwrap(); assert_eq!( std::fs::read_to_string(dest.join("pkg-1.0/docs/v1/manual.txt")).unwrap(), "content" ); // A real hard link shares the inode with its target use std::os::unix::fs::MetadataExt; assert_eq!( dest.join("pkg-1.0/manual.txt").metadata().unwrap().ino(), dest.join("pkg-1.0/docs/v1/manual.txt") .metadata() .unwrap() .ino() ); } #[test] fn test_extract_tar_skips_pax_global_header() { // Regression: git-archive tarballs open with a pax global header // record ('pax_global_header'): GNU tar consumes it and never // lists or extracts it, but the tar crate yields it as an entry. // Counting it for the wrapper detection made every git-archive // orig look multi-rooted: the unwrap was skipped, the sources // stayed nested one level too deep and the package's .diff.gz // did not apply (linux-riscv). let temp_dir = tempfile::tempdir().unwrap(); let tar_path = temp_dir.path().join("git.orig.tar"); let mut tar_bytes = Vec::new(); push_tar_entry( &mut tar_bytes, "pax_global_header", b'g', b"52 comment=d0e1c2d", None, ); push_tar_entry(&mut tar_bytes, "pkg-1.0/Makefile", b'0', b"all:", None); tar_bytes.extend_from_slice(&[0u8; 1024]); std::fs::write(&tar_path, tar_bytes).unwrap(); let tree = temp_dir.path().join("tree"); let extracted = extract_tar_archive(&tar_path, &tree, &crate::report::Quiet, |f| f).unwrap(); unwrap_wrapper(&tree, &extracted.wrapper.unwrap()).unwrap(); assert!(!tree.join("pax_global_header").exists()); assert_eq!( std::fs::read_to_string(tree.join("Makefile")).unwrap(), "all:" ); } #[test] fn test_extract_tar_rejects_hardlink_escape() { let temp_dir = tempfile::tempdir().unwrap(); let tar_path = temp_dir.path().join("malicious.orig.tar"); let mut tar_bytes = Vec::new(); push_tar_entry(&mut tar_bytes, "good.txt", b'0', b"ok", None); push_tar_entry(&mut tar_bytes, "evil.txt", b'1', b"", Some("../evil.txt")); tar_bytes.extend_from_slice(&[0u8; 1024]); std::fs::write(&tar_path, tar_bytes).unwrap(); let dest = temp_dir.path().join("dest"); let result = extract_tar_archive(&tar_path, &dest, &crate::report::Quiet, |f| f); let err = match result { Ok(_) => panic!("extraction of an escaping hard link should fail"), Err(e) => e, }; assert!( err.to_string().contains("../evil.txt"), "error should name the offending link target, got: {err}" ); assert!(!temp_dir.path().join("evil.txt").exists()); assert_eq!( std::fs::read_to_string(dest.join("good.txt")).unwrap(), "ok" ); } #[test] fn test_unwrap_wrapper_without_dir_entries() { // Regression: tarballs carrying no explicit entries for their // directories (e.g. the 'tree' orig) used to skip the relocation // entirely, leaving the sources nested one level too deep and // dpkg-source rejecting the tree as 'unexpected upstream changes' let temp_dir = tempfile::tempdir().unwrap(); let tar_path = temp_dir.path().join("wrapped.orig.tar"); std::fs::write( &tar_path, build_tar(&[ ("tree-2.3.2/Makefile", b"all:"), ("tree-2.3.2/doc/tree.1", b"th"), ]), ) .unwrap(); let tree = temp_dir.path().join("tree"); let extracted = extract_tar_archive(&tar_path, &tree, &crate::report::Quiet, |f| f).unwrap(); unwrap_wrapper(&tree, &extracted.wrapper.unwrap()).unwrap(); assert_eq!( std::fs::read_to_string(tree.join("Makefile")).unwrap(), "all:" ); assert_eq!( std::fs::read_to_string(tree.join("doc/tree.1")).unwrap(), "th" ); assert!(!tree.join("tree-2.3.2").exists()); } #[test] fn test_unwrap_wrapper_preserves_symlinks() { // Regression: the copy-based relocation used to flatten top-level // symlinks into plain files (dpkg-source: 'cannot represent change // to ...: old version is symlink') let temp_dir = tempfile::tempdir().unwrap(); let tar_path = temp_dir.path().join("wrapped.orig.tar"); let mut tar_bytes = Vec::new(); push_tar_entry(&mut tar_bytes, "pkg-1.0/README.md", b'0', b"readme", None); push_tar_entry( &mut tar_bytes, "pkg-1.0/README", b'2', b"", Some("README.md"), ); tar_bytes.extend_from_slice(&[0u8; 1024]); std::fs::write(&tar_path, tar_bytes).unwrap(); let tree = temp_dir.path().join("tree"); let extracted = extract_tar_archive(&tar_path, &tree, &crate::report::Quiet, |f| f).unwrap(); unwrap_wrapper(&tree, &extracted.wrapper.unwrap()).unwrap(); let meta = std::fs::symlink_metadata(tree.join("README")).unwrap(); assert!(meta.file_type().is_symlink()); assert_eq!( std::fs::read_link(tree.join("README")).unwrap(), Path::new("README.md") ); assert_eq!( std::fs::read_to_string(tree.join("README")).unwrap(), "readme" ); } #[test] fn test_unwrap_wrapper_leaves_flat_layout() { let temp_dir = tempfile::tempdir().unwrap(); let tar_path = temp_dir.path().join("flat.orig.tar"); std::fs::write( &tar_path, build_tar(&[("Makefile", b"all:"), ("debian/rules", b"#!/m")]), ) .unwrap(); let tree = temp_dir.path().join("tree"); let extracted = extract_tar_archive(&tar_path, &tree, &crate::report::Quiet, |f| f).unwrap(); assert!(extracted.wrapper.is_none()); assert!(tree.join("Makefile").exists()); assert!(tree.join("debian/rules").exists()); } #[test] fn test_unwrap_wrapper_of_second_multi_orig_component() { // Regression: multi-orig packages (e.g. dovecot) extract a second // component into a tree root the first component already filled, // and dpkg-source expects the component's content at // '/' ('pigeonhole' for '...orig-pigeonhole.tar.gz'), // not the wrapper's own layout at the root. The addon directory // used to never be created: every move into it failed with ENOENT // and the pull aborted. let temp_dir = tempfile::tempdir().unwrap(); let tree = temp_dir.path().join("tree"); let main = temp_dir.path().join("main.orig.tar"); std::fs::write(&main, build_tar(&[("dovecot-2.4.5/README", b"main")])).unwrap(); let extracted = extract_tar_archive(&main, &tree, &crate::report::Quiet, |f| f).unwrap(); unwrap_into( &tree, &extracted.wrapper.unwrap(), component_addon("main.orig.tar"), ) .unwrap(); let component = temp_dir.path().join("pkg_1.0.orig-pigeonhole.tar.gz"); std::fs::write( &component, build_tar(&[ ("dovecot-pigeonhole-2.4.5/update-version.sh", b"sh"), ("dovecot-pigeonhole-2.4.5/pigeonhole/x", b"ph"), ]), ) .unwrap(); let extracted = extract_tar_archive(&component, &tree, &crate::report::Quiet, |f| f).unwrap(); unwrap_into( &tree, &extracted.wrapper.unwrap(), component_addon("pkg_1.0.orig-pigeonhole.tar.gz"), ) .unwrap(); assert_eq!( std::fs::read_to_string(tree.join("README")).unwrap(), "main" ); assert_eq!( std::fs::read_to_string(tree.join("pigeonhole/update-version.sh")).unwrap(), "sh" ); assert_eq!( std::fs::read_to_string(tree.join("pigeonhole/pigeonhole/x")).unwrap(), "ph" ); assert!(!tree.join("update-version.sh").exists()); assert!(!tree.join("dovecot-2.4.5").exists()); assert!(!tree.join("dovecot-pigeonhole-2.4.5").exists()); } #[test] fn test_unwrap_component_already_prefixed_with_addon() { // Regression: a component tarball whose entries already carry the // addon prefix ('www/...' in 'sqlite3_3.53.4.orig-www.tar.xz') // extracts to its end state. It used to be treated as a wrapper of // the same name as the addon: unwrap_into cleared the target — // destroying the component just extracted into it — and unwrapped // the emptied directory into itself, leaving no 'www' at all. let temp_dir = tempfile::tempdir().unwrap(); let tar_path = temp_dir.path().join("pkg_1.0.orig-www.tar.gz"); std::fs::write( &tar_path, build_tar(&[("www/README", b"docs"), ("www/img/x", b"img")]), ) .unwrap(); let tree = temp_dir.path().join("tree"); let extracted = extract_tar_archive(&tar_path, &tree, &crate::report::Quiet, |f| f).unwrap(); unwrap_into( &tree, &extracted.wrapper.unwrap(), component_addon("pkg_1.0.orig-www.tar.gz"), ) .unwrap(); assert_eq!( std::fs::read_to_string(tree.join("www/README")).unwrap(), "docs" ); assert_eq!( std::fs::read_to_string(tree.join("www/img/x")).unwrap(), "img" ); } /// A minimal ustar archive of `pkg-1.0/{Makefile,doc/readme}`, /// bz2-compressed. const BZ2_TAR_WRAPPED: &[u8] = &[ 66, 90, 104, 57, 49, 65, 89, 38, 83, 89, 88, 108, 110, 100, 0, 0, 154, 255, 128, 201, 128, 0, 16, 64, 3, 231, 144, 0, 3, 64, 0, 111, 174, 222, 0, 8, 40, 32, 0, 116, 18, 148, 218, 147, 200, 67, 33, 166, 100, 70, 65, 250, 83, 212, 18, 145, 1, 166, 128, 0, 0, 208, 62, 196, 104, 44, 169, 64, 3, 160, 68, 65, 10, 40, 112, 177, 128, 161, 163, 8, 68, 16, 80, 10, 79, 36, 5, 53, 192, 112, 39, 81, 227, 129, 9, 73, 68, 32, 93, 136, 243, 37, 49, 35, 19, 133, 118, 14, 245, 37, 8, 233, 250, 99, 27, 194, 182, 183, 86, 186, 163, 237, 147, 31, 53, 154, 53, 20, 231, 189, 6, 40, 100, 34, 7, 241, 119, 36, 83, 133, 9, 5, 134, 198, 230, 64, ]; #[test] fn test_extract_tar_bz2() { // Regression: '.tar.bz2' origs (e.g. busybox) were downloaded but // never extracted — the dispatch only knew gz and xz — leaving a // tree that dpkg-source packed as a mass of ignored deletions let temp_dir = tempfile::tempdir().unwrap(); let tar_path = temp_dir.path().join("wrapped.orig.tar.bz2"); std::fs::write(&tar_path, BZ2_TAR_WRAPPED).unwrap(); let tree = temp_dir.path().join("tree"); let extracted = extract_tar_archive( &tar_path, &tree, &crate::report::Quiet, bzip2_rs::DecoderReader::new, ) .unwrap(); unwrap_wrapper(&tree, &extracted.wrapper.unwrap()).unwrap(); assert_eq!( std::fs::read_to_string(tree.join("Makefile")).unwrap(), "all:" ); assert_eq!( std::fs::read_to_string(tree.join("doc/readme")).unwrap(), "docs" ); assert!(!tree.join("pkg-1.0").exists()); } }