vhaudiquet
37e0b5c978
Sources listed with "3.0 (quilt)" extra components (node-jest, php-*, ...) carry one tarball per bundled module next to the main orig, named <package>_<uver>.orig-<component>.tar.<ext>. fetch_orig_tarball picked the single file matching ".orig.tar." — which cannot even match the component naming — so a git pull only fetched the main orig. The later dpkg-source -b quilt verification then failed with "can't find file to patch" on the first patch touching a component directory. Select the files with the existing build::changes::is_orig_tarball helper (mirroring dpkg's \.orig(-.+)?\.tar\. strip pattern) and fetch all of them, pristine-tar checkout first with a checksummed archive download fallback, per tarball. The end-to-end test now asserts every stanza-listed orig lands in the package dir instead of just any *.orig.tar.* file, and gains a node-jest (trixie, 24 components) regression case. Verified live: pkh pull node-jest -d debian fetches all 15 origs of the sid ds7 repack, and dpkg-source -b builds the debian.tar.xz and dsc without touching the series.
1085 lines
38 KiB
Rust
1085 lines
38 KiB
Rust
use std::cmp::min;
|
|
use std::error::Error;
|
|
use std::os::unix::fs::symlink;
|
|
use std::path::Component;
|
|
use std::path::Path;
|
|
use std::path::PathBuf;
|
|
|
|
use crate::package_info::PackageInfo;
|
|
|
|
use std::process::Command;
|
|
|
|
use log::debug;
|
|
|
|
use regex::Regex;
|
|
|
|
use crate::ProgressCallback;
|
|
|
|
fn clone_repo(
|
|
url: &str,
|
|
package: &str,
|
|
branch: Option<&str>,
|
|
cwd: Option<&Path>,
|
|
progress: ProgressCallback<'_>,
|
|
) -> Result<(), Box<dyn Error>> {
|
|
let target_path = if let Some(path) = cwd {
|
|
path.join(package)
|
|
} else {
|
|
Path::new(package).to_path_buf()
|
|
};
|
|
|
|
let mut callbacks = git2::RemoteCallbacks::new();
|
|
if let Some(ref progress_cb) = progress {
|
|
// Download progress
|
|
callbacks.transfer_progress(move |stats| {
|
|
(progress_cb)(
|
|
"",
|
|
"Receiving objects...",
|
|
stats.received_objects(),
|
|
stats.total_objects(),
|
|
);
|
|
true
|
|
});
|
|
|
|
// Remote progress: messages 'Remote: compressing objects 10% (34/340)'
|
|
// Parse progress informations to display them in callbacks
|
|
callbacks.sideband_progress(move |data| {
|
|
let msg = String::from_utf8_lossy(data);
|
|
let re = Regex::new(r"(.*):[ ]*([0-9]*)% \(([0-9]*)/([0-9]*)\)").unwrap();
|
|
if let Some(caps) = re.captures(msg.trim()) {
|
|
let msg = caps.get(1).map_or("", |m| m.as_str()).to_string();
|
|
let objects = caps
|
|
.get(3)
|
|
.map_or("", |m| m.as_str())
|
|
.to_string()
|
|
.parse::<usize>()
|
|
.unwrap_or(0);
|
|
let total = caps
|
|
.get(4)
|
|
.map_or("", |m| m.as_str())
|
|
.to_string()
|
|
.parse::<usize>()
|
|
.unwrap_or(0);
|
|
|
|
(progress_cb)("", msg.as_str(), objects, total);
|
|
}
|
|
|
|
true
|
|
});
|
|
}
|
|
|
|
let mut fetch_options = git2::FetchOptions::new();
|
|
fetch_options.remote_callbacks(callbacks);
|
|
|
|
let mut builder = git2::build::RepoBuilder::new();
|
|
builder.fetch_options(fetch_options);
|
|
|
|
if let Some(b) = branch {
|
|
builder.branch(b);
|
|
}
|
|
|
|
match builder.clone(url, &target_path) {
|
|
Ok(_repo) => Ok(()),
|
|
Err(e) => Err(format!("Failed to clone: {}", e).into()),
|
|
}
|
|
}
|
|
|
|
use std::fs::File;
|
|
use std::io::Write;
|
|
|
|
use flate2::read::GzDecoder;
|
|
use futures_util::StreamExt;
|
|
use tar::Archive;
|
|
use xz2::read::XzDecoder;
|
|
|
|
fn copy_dir_all(src: &Path, dst: &Path) -> Result<(), Box<dyn Error>> {
|
|
if !dst.exists() {
|
|
std::fs::create_dir_all(dst)?;
|
|
}
|
|
|
|
for entry in std::fs::read_dir(src)? {
|
|
let entry = entry?;
|
|
let src_path = entry.path();
|
|
let dst_path = dst.join(entry.file_name());
|
|
|
|
// Reproduce symlinks as symlinks rather than following them, so that
|
|
// dangling/absolute symlinks do not abort the copy.
|
|
if std::fs::symlink_metadata(&src_path)?
|
|
.file_type()
|
|
.is_symlink()
|
|
{
|
|
let target = std::fs::read_link(&src_path)?;
|
|
let _ = std::fs::remove_file(&dst_path);
|
|
symlink(&target, &dst_path)?;
|
|
} else if src_path.is_dir() {
|
|
copy_dir_all(&src_path, &dst_path)?;
|
|
} else {
|
|
// Copy the file, preserving the source modification and access
|
|
// times. This is important for autotools/gnulib-based packages
|
|
// (e.g. 'hello' from Debian sid) that ship pre-generated files
|
|
// alongside their prerequisites: if the copy resets the mtime to
|
|
// "now", the prerequisites appear as new as the generated targets
|
|
// and `make` tries to regenerate them using tools (like gperf)
|
|
// that are not declared build-dependencies.
|
|
std::fs::copy(&src_path, &dst_path)?;
|
|
copy_file_times(&src_path, &dst_path)?;
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Restore the modification and access times of `dest` to match `src`.
|
|
///
|
|
/// `std::fs::copy` resets the destination mtime to "now", which breaks
|
|
/// timestamp-based build systems (autotools/gnulib) that ship pre-generated
|
|
/// files alongside their prerequisites. Restoring the original timestamps
|
|
/// prevents `make` from needlessly regenerating those files with tools that
|
|
/// may not be installed (e.g. `gperf`).
|
|
fn copy_file_times(src: &Path, dest: &Path) -> Result<(), Box<dyn Error>> {
|
|
let metadata = std::fs::metadata(src)?;
|
|
let mut times = std::fs::FileTimes::new();
|
|
let mut have_times = false;
|
|
if let Ok(mtime) = metadata.modified() {
|
|
times = times.set_modified(mtime);
|
|
have_times = true;
|
|
}
|
|
if let Ok(atime) = metadata.accessed() {
|
|
times = times.set_accessed(atime);
|
|
have_times = true;
|
|
}
|
|
if have_times && let Ok(dest_file) = std::fs::File::open(dest) {
|
|
let _ = dest_file.set_times(times);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Result of extracting an archive
|
|
struct ExtractedArchive {
|
|
/// Paths of the extracted files
|
|
files: Vec<String>,
|
|
/// The archive used './'-prefixed entries: its contents were extracted
|
|
/// directly into the destination directory, and there is no
|
|
/// 'package-version/' top-level directory to relocate
|
|
in_place: bool,
|
|
}
|
|
|
|
/// Helper function to extract tar archive with progress tracking
|
|
fn extract_tar_archive<D, F>(
|
|
file_path: &Path,
|
|
dest: &Path,
|
|
progress: ProgressCallback<'_>,
|
|
decoder_factory: F,
|
|
) -> Result<ExtractedArchive, Box<dyn Error>>
|
|
where
|
|
D: std::io::Read,
|
|
F: Fn(File) -> D,
|
|
{
|
|
let file = File::open(file_path)?;
|
|
let decoder = decoder_factory(file);
|
|
let mut archive = Archive::new(decoder);
|
|
|
|
// Get total number of entries for progress tracking
|
|
let total_entries = archive.entries()?.count();
|
|
let mut current_entry = 0;
|
|
|
|
// Reset the archive to read entries again
|
|
let file = File::open(file_path)?;
|
|
let decoder = decoder_factory(file);
|
|
let mut archive = Archive::new(decoder);
|
|
|
|
let mut extracted_files = Vec::new();
|
|
let mut in_place = false;
|
|
|
|
for entry in archive.entries()? {
|
|
let mut entry = entry?;
|
|
let path = entry.path()?.to_path_buf();
|
|
|
|
// Archives built with './'-prefixed entries (common in third-party
|
|
// repositories) target the destination directory itself, with no
|
|
// 'package-version/' wrapper; skip their root entry, and remember
|
|
// the layout for the caller
|
|
let relative = if path.to_string_lossy().starts_with("./") {
|
|
in_place = true;
|
|
path.strip_prefix("./").unwrap_or(&path).to_path_buf()
|
|
} else {
|
|
path
|
|
};
|
|
|
|
if relative.as_os_str().is_empty() {
|
|
continue;
|
|
}
|
|
|
|
// Security: `Entry::unpack` performs no path sanitization, so extract
|
|
// the entry only if its path is confined to the destination directory.
|
|
// An absolute path or a path containing '..' would let a malicious (or
|
|
// deeply malformed) tarball write files anywhere outside 'dest' (path
|
|
// traversal). Refuse such entries with an error rather than skipping
|
|
// them silently, so the problem is not hidden.
|
|
let escapes_dest = relative.components().any(|component| {
|
|
matches!(
|
|
component,
|
|
Component::Prefix(_) | Component::RootDir | Component::ParentDir
|
|
)
|
|
});
|
|
if escapes_dest {
|
|
return Err(format!(
|
|
"Refusing to extract '{}': archive entry path is absolute or \
|
|
contains '..' and would escape the destination directory '{}'",
|
|
relative.display(),
|
|
dest.display()
|
|
)
|
|
.into());
|
|
}
|
|
|
|
let dest_path = dest.join(&relative);
|
|
|
|
// Create parent directories if needed
|
|
if let Some(parent) = dest_path.parent() {
|
|
std::fs::create_dir_all(parent)?;
|
|
}
|
|
|
|
// Extract the file
|
|
entry.unpack(&dest_path)?;
|
|
extracted_files.push(dest_path.to_string_lossy().to_string());
|
|
|
|
current_entry += 1;
|
|
|
|
// Report progress
|
|
if let Some(cb) = progress {
|
|
cb("", "Extracting...", current_entry, total_entries);
|
|
}
|
|
}
|
|
|
|
Ok(ExtractedArchive {
|
|
files: extracted_files,
|
|
in_place,
|
|
})
|
|
}
|
|
|
|
fn extract_archive(
|
|
path: &Path,
|
|
dest: &Path,
|
|
progress: ProgressCallback<'_>,
|
|
) -> Result<ExtractedArchive, Box<dyn Error>> {
|
|
let filename = path.file_name().unwrap().to_string_lossy();
|
|
|
|
if filename.ends_with(".tar.gz") || filename.ends_with(".tgz") {
|
|
extract_tar_archive(path, dest, progress, GzDecoder::new)
|
|
} else if filename.ends_with(".tar.xz") || filename.ends_with(".txz") {
|
|
extract_tar_archive(path, dest, progress, XzDecoder::new)
|
|
} else {
|
|
Err(format!("Unsupported archive format: {}", filename).into())
|
|
}
|
|
}
|
|
|
|
fn checkout_pristine_tar(package_dir: &Path, filename: &str) -> Result<(), Box<dyn Error>> {
|
|
let output = Command::new("pristine-tar")
|
|
.current_dir(package_dir)
|
|
.args(["checkout", format!("../{filename}").as_str()])
|
|
.output()
|
|
.map_err(|e| {
|
|
format!(
|
|
"Failed to run 'pristine-tar' to check out '{filename}': {}. \
|
|
Is 'pristine-tar' installed? It is required to reconstruct \
|
|
the upstream orig tarball from a git repository.",
|
|
e
|
|
)
|
|
})?;
|
|
|
|
if !output.status.success() {
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
return Err(format!(
|
|
"pristine-tar checkout of '{filename}' failed with status: {}.{}",
|
|
output.status,
|
|
if stderr.trim().is_empty() {
|
|
String::new()
|
|
} else {
|
|
format!("\npristine-tar output:\n{}", stderr.trim())
|
|
}
|
|
)
|
|
.into());
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
async fn download_file_checksum(
|
|
url: &str,
|
|
checksum: &str,
|
|
algo: crate::package_info::ChecksumAlgo,
|
|
target_dir: &Path,
|
|
progress: ProgressCallback<'_>,
|
|
) -> Result<(), Box<dyn Error>> {
|
|
// Archive mirrors and CDNs are busy enough that single attempts fail
|
|
// spuriously (dropped connections, truncated bodies, index generations
|
|
// momentarily out of sync): retry the whole download a few times before
|
|
// reporting the last failure.
|
|
const ATTEMPTS: u32 = 3;
|
|
let mut last_error: Box<dyn Error> = String::new().into();
|
|
for attempt in 1..=ATTEMPTS {
|
|
match download_file_checksum_once(url, checksum, algo, target_dir, progress).await {
|
|
Ok(()) => return Ok(()),
|
|
Err(e) => {
|
|
log::warn!("download of '{url}' failed (attempt {attempt}/{ATTEMPTS}): {e}");
|
|
last_error = e;
|
|
tokio::time::sleep(std::time::Duration::from_millis(500 * u64::from(attempt)))
|
|
.await;
|
|
}
|
|
}
|
|
}
|
|
Err(format!("downloading '{url}' failed after {ATTEMPTS} attempts: {last_error}").into())
|
|
}
|
|
|
|
/// One download attempt of [`download_file_checksum`], verifying the
|
|
/// content length and the expected checksum
|
|
async fn download_file_checksum_once(
|
|
url: &str,
|
|
checksum: &str,
|
|
algo: crate::package_info::ChecksumAlgo,
|
|
target_dir: &Path,
|
|
progress: ProgressCallback<'_>,
|
|
) -> Result<(), Box<dyn Error>> {
|
|
// Download with the shared client (connect timeout). Large orig tarballs
|
|
// can legitimately take longer than the client's default total timeout,
|
|
// so use a generous per-request timeout for streaming downloads
|
|
let response = crate::distro_info::http_get_retried_with_timeout(
|
|
url,
|
|
Some(std::time::Duration::from_secs(30 * 60)),
|
|
)
|
|
.await
|
|
.map_err(|e| Box::new(e) as Box<dyn Error>)?;
|
|
if !response.status().is_success() {
|
|
return Err(format!("Failed to download '{}' : {}", url, response.status()).into());
|
|
}
|
|
let total_size = response.content_length();
|
|
|
|
// Target file: extract file name from URL
|
|
let filename = Path::new(url)
|
|
.file_name()
|
|
.and_then(|f| f.to_str())
|
|
.ok_or_else(|| {
|
|
format!("Could not determine a file name from URL '{url}' to download the package file")
|
|
})?;
|
|
let path = target_dir.join(filename);
|
|
let mut file = File::create(path)?;
|
|
|
|
// Download chunk by chunk to disk, while updating hasher for checksum
|
|
let mut stream = response.bytes_stream();
|
|
// Accumulate the downloaded bytes so we can compute the final digest with the
|
|
// correct algorithm once the download is complete.
|
|
let mut buffer: Vec<u8> = Vec::with_capacity(total_size.unwrap_or(0) as usize);
|
|
while let Some(item) = stream.next().await {
|
|
let chunk = item?;
|
|
file.write_all(&chunk)?;
|
|
buffer.extend_from_slice(&chunk);
|
|
|
|
if let (Some(cb), Some(total)) = (progress, total_size) {
|
|
let index = min(buffer.len(), total as usize);
|
|
cb("", "Downloading...", index, total as usize);
|
|
}
|
|
}
|
|
|
|
// A dropped connection can end the stream early: never hand a truncated
|
|
// file to the checksum check (its mismatch message would hide the cause)
|
|
if let Some(total) = total_size
|
|
&& buffer.len() != total as usize
|
|
{
|
|
return Err(format!(
|
|
"incomplete download from '{url}': got {} of {total} bytes",
|
|
buffer.len()
|
|
)
|
|
.into());
|
|
}
|
|
|
|
// Verify checksum using the algorithm specified for this file
|
|
let calculated_checksum = algo.hex_digest(&buffer);
|
|
if calculated_checksum != checksum {
|
|
return Err(format!(
|
|
"Checksum mismatch! Expected {}, got {}",
|
|
checksum, calculated_checksum
|
|
)
|
|
.into());
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
fn setup_pristine_tar_branch(package_dir: &Path, dist: &str) -> Result<(), Box<dyn Error>> {
|
|
let repo = git2::Repository::open(package_dir)?;
|
|
|
|
// Check if local branch already exists
|
|
if repo
|
|
.find_branch("pristine-tar", git2::BranchType::Local)
|
|
.is_ok()
|
|
{
|
|
return Ok(());
|
|
}
|
|
|
|
// Find remote pristine-tar branch
|
|
let branches = repo.branches(Some(git2::BranchType::Remote))?;
|
|
for branch_result in branches {
|
|
let (branch, _) = branch_result?;
|
|
if let Some(name) = branch.name()?
|
|
&& name.ends_with(&format!("/{dist}/pristine-tar"))
|
|
{
|
|
debug!("Found remote pristine-tar branch: {}", name);
|
|
|
|
let commit = branch.get().peel_to_commit()?;
|
|
|
|
// Create local branch
|
|
let mut local_branch = repo.branch("pristine-tar", &commit, false)?;
|
|
|
|
// Set upstream
|
|
local_branch.set_upstream(Some(name))?;
|
|
|
|
debug!("Created local pristine-tar branch tracking {}", name);
|
|
return Ok(());
|
|
}
|
|
}
|
|
|
|
debug!("No remote pristine-tar branch found.");
|
|
Ok(())
|
|
}
|
|
|
|
async fn fetch_orig_tarball(
|
|
info: &PackageInfo,
|
|
cwd: Option<&Path>,
|
|
progress: ProgressCallback<'_>,
|
|
) -> Result<(), Box<dyn Error>> {
|
|
let package_dir = if let Some(path) = cwd {
|
|
path.join(&info.stanza.package)
|
|
} else {
|
|
Path::new(&info.stanza.package).to_path_buf()
|
|
};
|
|
|
|
// Upstream tarballs in the file list: the main orig tarball plus, for
|
|
// multi-orig ("3.0 (quilt)" extra-component) sources, one component
|
|
// tarball per bundled module (`*.orig-<component>.tar.<ext>`). dpkg-source
|
|
// unpacks all of them side by side, so a git pull must fetch them all.
|
|
let orig_files: Vec<_> = info
|
|
.stanza
|
|
.files
|
|
.iter()
|
|
.filter(|f| crate::build::changes::is_orig_tarball(&f.name))
|
|
.collect();
|
|
if orig_files.is_empty() {
|
|
return Err(format!(
|
|
"Could not find orig tarball in file list for package '{}'. \
|
|
Available files: {:?}",
|
|
info.stanza.package,
|
|
info.stanza
|
|
.files
|
|
.iter()
|
|
.map(|f| &f.name)
|
|
.collect::<Vec<_>>()
|
|
)
|
|
.into());
|
|
}
|
|
|
|
// 1. Try executing pristine-tar
|
|
|
|
// Setup pristine-tar branch if needed (by tracking remote branch)
|
|
let _ = setup_pristine_tar_branch(&package_dir, info.dist.as_str());
|
|
|
|
for orig_file in orig_files {
|
|
let filename = &orig_file.name;
|
|
|
|
if let Err(e) = checkout_pristine_tar(&package_dir, filename.as_str()) {
|
|
debug!(
|
|
"pristine-tar failed: {}. Falling back to archive download.",
|
|
e
|
|
);
|
|
|
|
// 2. Fallback to archive download
|
|
// We download to the parent directory of the package repo (which is standard for build tools)
|
|
// or the current directory if cwd is None (which effectively is the parent of the package dir)
|
|
let target_dir = cwd.unwrap_or_else(|| Path::new("."));
|
|
download_file_checksum(
|
|
format!("{}/{}", info.archive_url, filename).as_str(),
|
|
&orig_file.checksum,
|
|
orig_file.checksum_algo,
|
|
target_dir,
|
|
progress,
|
|
)
|
|
.await?;
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
async fn fetch_dsc_file(
|
|
info: &PackageInfo,
|
|
cwd: Option<&Path>,
|
|
progress: ProgressCallback<'_>,
|
|
) -> Result<(), Box<dyn Error>> {
|
|
let target_dir = cwd.unwrap_or_else(|| Path::new("."));
|
|
|
|
// Find the dsc file in the file list
|
|
let dsc_file = info
|
|
.stanza
|
|
.files
|
|
.iter()
|
|
.find(|f| f.name.ends_with(".dsc"))
|
|
.ok_or("Could not find .dsc file in package info")?;
|
|
let filename = &dsc_file.name;
|
|
|
|
debug!("Fetching dsc file: {}", filename);
|
|
|
|
download_file_checksum(
|
|
format!("{}/{}", info.archive_url, filename).as_str(),
|
|
&dsc_file.checksum,
|
|
dsc_file.checksum_algo,
|
|
target_dir,
|
|
progress,
|
|
)
|
|
.await?;
|
|
|
|
Ok(())
|
|
}
|
|
|
|
async fn fetch_archive_sources(
|
|
info: &PackageInfo,
|
|
cwd: Option<&Path>,
|
|
progress: ProgressCallback<'_>,
|
|
) -> Result<(), Box<dyn Error>> {
|
|
let package_dir = if let Some(path) = cwd {
|
|
path
|
|
} else {
|
|
&Path::new(".").to_path_buf()
|
|
};
|
|
|
|
std::fs::create_dir_all(package_dir)?;
|
|
|
|
for file in &info.stanza.files {
|
|
let url = format!("{}/{}", info.archive_url, file.name);
|
|
download_file_checksum(
|
|
&url,
|
|
&file.checksum,
|
|
file.checksum_algo,
|
|
package_dir,
|
|
progress,
|
|
)
|
|
.await?;
|
|
|
|
// Extract all tar archives, merging extracted directories
|
|
if file.name.ends_with(".tar.gz") || file.name.ends_with(".tar.xz") {
|
|
let path = package_dir.join(&file.name);
|
|
let extract_dir = package_dir.join(&info.stanza.package);
|
|
|
|
let extracted = extract_archive(&path, &extract_dir, progress)?;
|
|
|
|
// Special case: the debian tar does only contain 'debian'
|
|
if file.name.contains("debian.tar.") {
|
|
continue;
|
|
}
|
|
|
|
// Archives with './'-prefixed entries are already laid out
|
|
// directly in the package directory; only 'package-version/'
|
|
// style ones need their contents relocated
|
|
if !extracted.in_place {
|
|
// List root directories extracted and use the first one as the source directory
|
|
debug!("Root directories extracted:");
|
|
let mut source_dir: Option<PathBuf> = None;
|
|
for file in &extracted.files {
|
|
let path = Path::new(file);
|
|
// Check if this is a directory and is at the archive root level
|
|
// (i.e., the path relative to extract_dir has no parent components)
|
|
if let Ok(relative_path) = path.strip_prefix(&extract_dir)
|
|
&& relative_path.components().count() == 1
|
|
&& path.is_dir()
|
|
{
|
|
debug!("- {}", relative_path.file_name().unwrap().to_string_lossy());
|
|
// Use the first directory found as the source
|
|
if source_dir.is_none() {
|
|
source_dir = Some(path.to_path_buf());
|
|
}
|
|
}
|
|
}
|
|
|
|
// Use the extracted directory as the source, assuming there is only one
|
|
if let Some(src_dir) = source_dir {
|
|
let target_dir = package_dir.join(&info.stanza.package);
|
|
|
|
if target_dir.exists() {
|
|
// Target exists, we need to merge contents
|
|
for sub_entry in std::fs::read_dir(&src_dir)? {
|
|
let sub_entry = sub_entry?;
|
|
let sub_path = sub_entry.path();
|
|
let target_path = target_dir.join(sub_entry.file_name());
|
|
if sub_path.is_dir() {
|
|
std::fs::create_dir_all(&target_path)?;
|
|
// Recursively copy directory contents
|
|
copy_dir_all(&sub_path, &target_path)?;
|
|
} else {
|
|
std::fs::copy(&sub_path, &target_path)?;
|
|
copy_file_times(&sub_path, &target_path)?;
|
|
}
|
|
}
|
|
std::fs::remove_dir_all(&src_dir)?;
|
|
} else {
|
|
std::fs::rename(&src_dir, &target_dir)?;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Extract and apply .diff.gz if present (old packages)
|
|
if file.name.ends_with(".diff.gz") {
|
|
let diff_gz_path = package_dir.join(&file.name);
|
|
let source_dir = package_dir.join(&info.stanza.package);
|
|
|
|
// Create the .diff file path by replacing .gz with empty string
|
|
let diff_path = diff_gz_path.with_extension("");
|
|
|
|
// Decompress the .diff.gz file directly to .diff
|
|
let input_file = File::open(&diff_gz_path)?;
|
|
let mut decoder = GzDecoder::new(input_file);
|
|
let mut output_file = File::create(&diff_path)?;
|
|
|
|
std::io::copy(&mut decoder, &mut output_file)?;
|
|
|
|
// Use relative path for the diff file (it's in the parent directory)
|
|
let relative_diff_path =
|
|
format!("../{}", diff_path.file_name().unwrap().to_string_lossy());
|
|
|
|
// Apply the patch using the patch command with relative path
|
|
let output = Command::new("patch")
|
|
.current_dir(&source_dir)
|
|
.arg("-p1")
|
|
.arg("--input")
|
|
.arg(&relative_diff_path)
|
|
.output()?;
|
|
|
|
if !output.status.success() {
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
return Err(
|
|
format!("Failed to apply patch: {}\n{}", diff_path.display(), stderr).into(),
|
|
);
|
|
}
|
|
|
|
debug!("Successfully applied patch: {}", diff_path.display());
|
|
|
|
// Clean up the extracted .diff file
|
|
std::fs::remove_file(&diff_path)?;
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Pull a source package locally using pre-retrieved package information
|
|
///
|
|
/// This function takes a PackageInfo struct and downloads the package using the preferred method
|
|
/// (either git or direct archive download), as well as orig tarball, inside 'package' directory.
|
|
/// The source will be extracted under 'package/package'.
|
|
pub async fn pull(
|
|
package_info: &PackageInfo,
|
|
cwd: Option<&Path>,
|
|
progress: ProgressCallback<'_>,
|
|
force_archive: bool,
|
|
) -> Result<(), Box<dyn Error>> {
|
|
let package = &package_info.stanza.package;
|
|
let series = &package_info.series;
|
|
let package_dir = if let Some(path) = cwd {
|
|
path.join(package)
|
|
} else {
|
|
Path::new(package).to_path_buf()
|
|
};
|
|
|
|
if package_dir.exists() {
|
|
return Err(format!(
|
|
"Directory '{}' already exists. Remove it first, or change working directory.",
|
|
package_dir.display()
|
|
)
|
|
.into());
|
|
}
|
|
|
|
/* Fetch the package: either via git (preferred VCS) or the archive */
|
|
if let Some(ref url) = package_info.preferred_vcs
|
|
&& !force_archive
|
|
{
|
|
// We have found a preferred VCS (git repository) for the package, so
|
|
// we fetch the package from that repo.
|
|
|
|
// Depending on target series, we pick target branch; if latest series is specified,
|
|
// we target the development branch, i.e. the default branch
|
|
// Only use Ubuntu-specific branch naming if the VCS is from Launchpad
|
|
let is_launchpad_vcs = url.contains("launchpad.net");
|
|
let series_list =
|
|
crate::distro_info::get_ordered_series_name(package_info.dist.as_str()).await?;
|
|
let latest_series = series_list.first().ok_or_else(|| {
|
|
format!(
|
|
"No series information available for distribution '{}', \
|
|
cannot determine its development series to select the git branch. \
|
|
The 'distro-info' package provides this data.",
|
|
package_info.dist
|
|
)
|
|
})?;
|
|
let branch_name = if latest_series != series {
|
|
if package_info.dist == "ubuntu" && is_launchpad_vcs {
|
|
Some(format!("{}/{}", package_info.dist, series))
|
|
} else {
|
|
// Debian does not have reliable branch naming...
|
|
// Also, Ubuntu packages with salsa VCS don't have Ubuntu-specific branches
|
|
// For now, we skip that part and clone default
|
|
// TODO: Inspect remote branches and tags for matches
|
|
None
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
|
|
if let Some(cb) = progress {
|
|
cb(
|
|
&format!(
|
|
"Cloning {}{}...",
|
|
url,
|
|
if let Some(b) = &branch_name {
|
|
format!(" (branch {})", b)
|
|
} else {
|
|
String::new()
|
|
}
|
|
),
|
|
"",
|
|
0,
|
|
0,
|
|
);
|
|
}
|
|
|
|
clone_repo(
|
|
url.as_str(),
|
|
package,
|
|
branch_name.as_deref(),
|
|
Some(&package_dir),
|
|
progress,
|
|
)?;
|
|
|
|
if !package_info.is_native() {
|
|
if let Some(cb) = progress {
|
|
cb("Fetching orig tarball...", "", 0, 0);
|
|
}
|
|
fetch_orig_tarball(package_info, Some(&package_dir), progress).await?;
|
|
} else {
|
|
debug!("Native package, skipping orig tarball fetch.");
|
|
}
|
|
|
|
if let Some(cb) = progress {
|
|
cb("Fetching dsc file...", "", 0, 0);
|
|
}
|
|
fetch_dsc_file(package_info, Some(&package_dir), progress).await?;
|
|
} else {
|
|
// Fallback to archive fetching
|
|
if let Some(cb) = progress {
|
|
cb("Downloading from archive...", "", 0, 0);
|
|
}
|
|
fetch_archive_sources(package_info, Some(&package_dir), progress).await?;
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
async fn test_pull_package_end_to_end(
|
|
package: &str,
|
|
series: Option<&str>,
|
|
dist: Option<&str>,
|
|
archive: Option<bool>,
|
|
) {
|
|
// This test verifies that 'pkh pull' clones the repo and fetches the tarball.
|
|
|
|
// For determinism, we require for tests that either a distro or series is specified,
|
|
// as no distribution would mean fallback to system distro
|
|
assert!(dist.is_some() || series.is_some());
|
|
|
|
// Use a temp directory as working directory
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let cwd = temp_dir.path();
|
|
|
|
// Main 'pull' command: the one we want to test
|
|
let info = crate::package_info::lookup(package, None, series, "", dist, None, None, None)
|
|
.await
|
|
.unwrap();
|
|
pull(&info, Some(cwd), None, archive.unwrap_or(false))
|
|
.await
|
|
.unwrap();
|
|
|
|
let package_dir = cwd.join(package);
|
|
assert!(package_dir.exists());
|
|
let package_source_dir = package_dir.join(package);
|
|
assert!(
|
|
package_source_dir.exists(),
|
|
"Package git repo directory not created"
|
|
);
|
|
assert!(
|
|
package_source_dir.join("debian").exists(),
|
|
"debian directory not present"
|
|
);
|
|
|
|
if package_source_dir.join(".git").exists() {
|
|
// Verify we are on the correct branch
|
|
let repo = git2::Repository::open(&package_source_dir).unwrap();
|
|
let head = repo.head().unwrap();
|
|
let name = head.name().unwrap();
|
|
|
|
// Check if the VCS is from Launchpad - only Launchpad has Ubuntu-specific branches
|
|
let is_launchpad_vcs = info
|
|
.preferred_vcs
|
|
.as_ref()
|
|
.map(|url| url.contains("launchpad.net"))
|
|
.unwrap_or(false);
|
|
|
|
if let Some(s) = series {
|
|
// The local branch should be named dist/series
|
|
// We skip debian for now as it does not have a reliable naming scheme
|
|
// Also skip Ubuntu packages with non-Launchpad VCS (e.g., salsa.debian.org)
|
|
if info.dist == "ubuntu" && is_launchpad_vcs {
|
|
assert_eq!(name, format!("refs/heads/{0}/{s}", info.dist));
|
|
}
|
|
} else {
|
|
// The local branch should be named ubuntu/devel for Ubuntu
|
|
// Debian unfortunately does not have a reliable naming scheme
|
|
// Also skip Ubuntu packages with non-Launchpad VCS
|
|
// Given that there was no series specified, and this is a test,
|
|
// we require to have a distribution specified
|
|
if dist.unwrap() == "ubuntu" && is_launchpad_vcs {
|
|
assert_eq!(name, "refs/heads/ubuntu/devel");
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check for the orig tarballs in the package dir (only for non-native
|
|
// packages): every orig listed in the stanza must be present, including
|
|
// the component tarballs of multi-orig packages (dpkg-source needs them
|
|
// all to unpack the merged upstream tree)
|
|
let mut found_dsc = false;
|
|
for entry in std::fs::read_dir(&package_dir).unwrap() {
|
|
let entry = entry.unwrap();
|
|
let name = entry.file_name().to_string_lossy().to_string();
|
|
if name.ends_with(".dsc") {
|
|
found_dsc = true;
|
|
}
|
|
}
|
|
|
|
if !info.is_native() {
|
|
for file in &info.stanza.files {
|
|
if crate::build::changes::is_orig_tarball(&file.name) {
|
|
assert!(
|
|
package_dir.join(&file.name).exists(),
|
|
"Orig tarball '{}' not found in package dir",
|
|
file.name
|
|
);
|
|
}
|
|
}
|
|
}
|
|
assert!(found_dsc, "DSC file not found in package dir");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_pull_hello_ubuntu_end_to_end() {
|
|
test_pull_package_end_to_end("hello", Some("noble"), None, None).await;
|
|
}
|
|
#[tokio::test]
|
|
async fn test_pull_hello_debian_end_to_end() {
|
|
test_pull_package_end_to_end("hello", Some("bookworm"), None, None).await;
|
|
}
|
|
|
|
/// Specific test for a package using a .diff.gz, instead of .debian and .orig
|
|
#[tokio::test]
|
|
async fn test_pull_linux_riscv_ubuntu_end_to_end() {
|
|
test_pull_package_end_to_end("linux-riscv", Some("noble"), None, Some(true)).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_pull_2048_universe_ubuntu_end_to_end() {
|
|
test_pull_package_end_to_end("2048", Some("noble"), None, None).await;
|
|
}
|
|
#[tokio::test]
|
|
async fn test_pull_1oom_contrib_debian_end_to_end() {
|
|
test_pull_package_end_to_end("1oom", Some("trixie"), None, None).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_pull_agg_svn_fallback_ok() {
|
|
test_pull_package_end_to_end("agg", Some("trixie"), None, None).await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_pull_hello_debian_latest_end_to_end() {
|
|
test_pull_package_end_to_end("hello", None, Some("debian"), None).await;
|
|
}
|
|
#[tokio::test]
|
|
async fn test_pull_hello_ubuntu_latest_end_to_end() {
|
|
test_pull_package_end_to_end("hello", None, Some("ubuntu"), None).await;
|
|
}
|
|
|
|
/// Test for paraview - a package that has no Ubuntu Launchpad code,
|
|
/// only a debian salsa repo, even in Ubuntu.
|
|
/// Furthermore, paraview has a Vcs-Git value of:
|
|
/// Vcs-Git: https://salsa.debian.org/science-team/paraview.git -b debian/latest
|
|
/// Given that it is not only an url but also specifies a branch, it needs
|
|
/// special care, that this test ensures.
|
|
#[tokio::test]
|
|
async fn test_pull_paraview_ubuntu_end_to_end() {
|
|
test_pull_package_end_to_end("paraview", Some("noble"), None, None).await;
|
|
}
|
|
|
|
/// Multi-orig ("3.0 (quilt)" extra component) regression test: node-jest
|
|
/// ships its bundled modules as separate `*.orig-<component>.tar.xz`
|
|
/// tarballs next to the main orig. The git pull path must fetch every
|
|
/// component, or the later `dpkg-source -b` quilt verification fails
|
|
/// with "can't find file to patch" on the first patch touching a
|
|
/// component directory.
|
|
#[tokio::test]
|
|
async fn test_pull_node_jest_debian_end_to_end() {
|
|
test_pull_package_end_to_end("node-jest", Some("trixie"), None, None).await;
|
|
}
|
|
|
|
/// Build a minimal uncompressed ustar archive from (name, data) entries.
|
|
///
|
|
/// Raw header blocks are crafted instead of using `tar::Builder` because
|
|
/// the builder itself refuses entry names containing '..' or absolute
|
|
/// paths, which is exactly what the traversal tests need to exercise.
|
|
fn build_tar(entries: &[(&str, &[u8])]) -> Vec<u8> {
|
|
let mut out = Vec::new();
|
|
for (name, data) in entries {
|
|
let mut block = [0u8; 512];
|
|
block[..name.len()].copy_from_slice(name.as_bytes());
|
|
block[100..108].copy_from_slice(b"0000644\0");
|
|
block[124..136].copy_from_slice(format!("{:011o}\0", data.len()).as_bytes());
|
|
block[136..148].copy_from_slice(b"00000000000\0");
|
|
block[156] = b'0'; // regular file
|
|
block[257..263].copy_from_slice(b"ustar\0");
|
|
block[263..265].copy_from_slice(b"00");
|
|
|
|
// Checksum: sum of the header bytes with the checksum field
|
|
// (bytes 148..156) taken as spaces
|
|
let mut checksum: u32 = 0;
|
|
for (i, byte) in block.iter().enumerate() {
|
|
checksum += if (148..156).contains(&i) {
|
|
u32::from(b' ')
|
|
} else {
|
|
u32::from(*byte)
|
|
};
|
|
}
|
|
block[148..154].copy_from_slice(format!("{checksum:06o}").as_bytes());
|
|
block[154] = 0;
|
|
block[155] = b' ';
|
|
|
|
out.extend_from_slice(&block);
|
|
out.extend_from_slice(data);
|
|
let padding = (512 - (data.len() % 512)) % 512;
|
|
out.extend_from_slice(&vec![0u8; padding]);
|
|
}
|
|
// End-of-archive marker: two zero-filled blocks
|
|
out.extend_from_slice(&[0u8; 1024]);
|
|
out
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_tar_rejects_parent_dir_traversal() {
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let tar_path = temp_dir.path().join("malicious.orig.tar");
|
|
std::fs::write(
|
|
&tar_path,
|
|
build_tar(&[("good.txt", b"ok"), ("../evil.txt", b"pwned")]),
|
|
)
|
|
.unwrap();
|
|
|
|
let dest = temp_dir.path().join("dest");
|
|
let result = extract_tar_archive(&tar_path, &dest, None, |f| f);
|
|
|
|
let err = match result {
|
|
Ok(_) => panic!("extraction of a traversal archive should fail"),
|
|
Err(e) => e,
|
|
};
|
|
assert!(
|
|
err.to_string().contains("../evil.txt"),
|
|
"error should name the offending entry, got: {err}"
|
|
);
|
|
// Nothing may be written outside of the destination directory
|
|
assert!(!temp_dir.path().join("evil.txt").exists());
|
|
// Legitimate entries preceding the malicious one are still extracted
|
|
assert_eq!(
|
|
std::fs::read_to_string(dest.join("good.txt")).unwrap(),
|
|
"ok"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_tar_rejects_absolute_path() {
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let tar_path = temp_dir.path().join("malicious.orig.tar");
|
|
std::fs::write(
|
|
&tar_path,
|
|
build_tar(&[
|
|
("good.txt", b"ok"),
|
|
("/pkh_test_absolute_escape.txt", b"pwned"),
|
|
]),
|
|
)
|
|
.unwrap();
|
|
|
|
let dest = temp_dir.path().join("dest");
|
|
let result = extract_tar_archive(&tar_path, &dest, None, |f| f);
|
|
|
|
let err = match result {
|
|
Ok(_) => panic!("extraction of an absolute-path archive should fail"),
|
|
Err(e) => e,
|
|
};
|
|
assert!(
|
|
err.to_string().contains("pkh_test_absolute_escape.txt"),
|
|
"error should name the offending entry, got: {err}"
|
|
);
|
|
// Nothing may be written at the filesystem root
|
|
assert!(!Path::new("/pkh_test_absolute_escape.txt").exists());
|
|
assert_eq!(
|
|
std::fs::read_to_string(dest.join("good.txt")).unwrap(),
|
|
"ok"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_tar_archive_normal_entries() {
|
|
let temp_dir = tempfile::tempdir().unwrap();
|
|
let tar_path = temp_dir.path().join("normal.orig.tar");
|
|
// Raw names: 'tar::Builder' would normalize away the './' prefix
|
|
std::fs::write(
|
|
&tar_path,
|
|
build_tar(&[
|
|
("hello-1.0/file.txt", b"hello"),
|
|
("./debian/rules", b"#!/m"),
|
|
]),
|
|
)
|
|
.unwrap();
|
|
|
|
let dest = temp_dir.path().join("dest");
|
|
let extracted = extract_tar_archive(&tar_path, &dest, None, |f| f).unwrap();
|
|
|
|
assert_eq!(
|
|
std::fs::read_to_string(dest.join("hello-1.0/file.txt")).unwrap(),
|
|
"hello"
|
|
);
|
|
assert_eq!(
|
|
std::fs::read_to_string(dest.join("debian/rules")).unwrap(),
|
|
"#!/m"
|
|
);
|
|
assert!(extracted.in_place);
|
|
assert!(
|
|
extracted.files.contains(
|
|
&dest
|
|
.join("hello-1.0/file.txt")
|
|
.to_string_lossy()
|
|
.to_string()
|
|
)
|
|
);
|
|
assert!(
|
|
extracted
|
|
.files
|
|
.contains(&dest.join("debian/rules").to_string_lossy().to_string())
|
|
);
|
|
}
|
|
}
|