diff --git a/src/build/mod.rs b/src/build/mod.rs index 66857ca..bbac02a 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -1492,6 +1492,7 @@ Provides: virtual-thing (= 2.0), plain-virtual Some(dist), None, None, + None, )) .unwrap_or_else(|e| panic!("package lookup failed for {}: {}", package, e)); rt.block_on(crate::pull::pull( diff --git a/src/deb/mod.rs b/src/deb/mod.rs index fd7d92a..43a7e58 100644 --- a/src/deb/mod.rs +++ b/src/deb/mod.rs @@ -383,7 +383,7 @@ mod tests { log::info!("Pulling package {} from {}...", package, series); let package_info = - crate::package_info::lookup(package, None, Some(series), "", dist, None, None) + crate::package_info::lookup(package, None, Some(series), "", dist, None, None, None) .await .expect("Cannot lookup package information"); crate::pull::pull(&package_info, Some(cwd), None, true) diff --git a/src/main.rs b/src/main.rs index 54dab34..ff425b7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -48,6 +48,8 @@ fn main() { .arg(arg!(-v --version "Target package version").required(false)) .arg(arg!(--archive "Only use the archive to download package source, not git").required(false)) .arg(arg!(--ppa "Download the package from a specific PPA").required(false)) + .arg(arg!(--repository "Download the package from an external flat repository, given as its full suite URL (e.g. https://pkg.noctalia.dev/deb/resolute/)").required(false) + .conflicts_with("ppa")) .arg(arg!(-p --pocket "Target package distribution pocket (updates, security, proposed)").required(false)) .arg(arg!( "Target package")), ) @@ -138,6 +140,9 @@ fn main() { let dist = sub_matches.get_one::("dist").map(|s| s.as_str()); let version = sub_matches.get_one::("version").map(|s| s.as_str()); let ppa = sub_matches.get_one::("ppa").map(|s| s.as_str()); + let repository = sub_matches + .get_one::("repository") + .map(|s| s.as_str()); let pocket = sub_matches .get_one::("pocket") .map(|s| s.as_str()) @@ -166,6 +171,7 @@ fn main() { pocket, dist, base_url.as_deref(), + repository, Some(&progress_callback), ) .await?; diff --git a/src/package_info.rs b/src/package_info.rs index 2a651d5..f0c5046 100644 --- a/src/package_info.rs +++ b/src/package_info.rs @@ -2,6 +2,7 @@ use flate2::read::GzDecoder; use std::collections::HashMap; use std::error::Error; use std::io::Read; +use xz2::read::XzDecoder; use crate::ProgressCallback; use crossterm::style::Stylize; @@ -151,15 +152,29 @@ impl PackageInfo { } } +/// Decompress a package index payload, detecting the compression format from +/// its magic bytes: gzip ('1f 8b'), xz ('fd 7zXZ 00'), or plain text +fn decompress_index(data: &[u8]) -> Result> { + let mut decoded: Box = if data.starts_with(&[0x1f, 0x8b]) { + Box::new(GzDecoder::new(data)) + } else if data.starts_with(&[0xfd, 0x37, b'z', b'X', b'Z', 0x00]) { + Box::new(XzDecoder::new(data)) + } else { + return Ok(String::from_utf8(data.to_vec())?); + }; + + let mut s = String::new(); + decoded.read_to_string(&mut s)?; + Ok(s) +} + struct DebianSources { splitted_sources: std::str::Split<'static, &'static str>, } impl DebianSources { fn new(data: &[u8]) -> Result> { - // Gz-decode 'Sources.gz' file into a string, and split it on stanzas - let mut d = GzDecoder::new(data); - let mut s = String::new(); - d.read_to_string(&mut s)?; + // Decode the index into a string, and split it on stanzas + let s = decompress_index(data)?; // Convert the string to a static lifetime by leaking it let static_str = Box::leak(s.into_boxed_str()); @@ -423,6 +438,123 @@ async fn find_package( Err(format!("Package '{}' not found.", package_name).into()) } +/// Fetch the 'Release' file at the root of a flat repository, and return its suite name +/// +/// Flat repositories (e.g. 'https://pkg.example.org/deb/resolute/') serve the +/// Release file at their root, without any 'dists/' hierarchy. The suite is +/// read from the 'Codename' field, falling back to 'Suite'. +async fn get_flat_repo_series(base_url: &str) -> Result> { + let url = format!("{}/Release", base_url.trim_end_matches('/')); + let response = reqwest::get(&url).await?; + if !response.status().is_success() { + return Err(format!( + "No Release file at '{}' (HTTP {}) - is '{}' the suite URL of a flat repository? \ + External repositories must be passed as their full suite URL.", + url, + response.status(), + base_url + ) + .into()); + } + let content = response.text().await?; + + for field in ["Codename", "Suite"] { + let prefix = format!("{field}:"); + if let Some(line) = content.lines().find(|l| l.starts_with(&prefix)) { + return Ok(line[prefix.len()..].trim().to_string()); + } + } + + Err(format!("No Codename or Suite field in Release file at '{url}'").into()) +} + +/// Fetch the sources index of a flat repository +/// +/// Flat repositories are free to serve any compressed variant of the index +/// (or an uncompressed one), so try the usual candidates in turn. +async fn get_flat_repo_sources(base_url: &str) -> Result, Box> { + let base = base_url.trim_end_matches('/'); + let mut errors = Vec::new(); + for name in ["Sources.xz", "Sources.gz", "Sources"] { + let url = format!("{base}/{name}"); + match reqwest::get(&url).await { + Ok(response) if response.status().is_success() => { + return Ok(response.bytes().await?.to_vec()); + } + Ok(response) => errors.push(format!("{}: HTTP {}", url, response.status())), + Err(e) => errors.push(format!("{}: {}", url, e)), + } + } + + Err(format!( + "Could not fetch the sources index from '{}': {}", + base, + errors.join(", ") + ) + .into()) +} + +/// Lookup package information in an external flat repository +/// +/// The URL points at the repository suite directory itself (e.g. +/// 'https://pkg.noctalia.dev/deb/resolute/'), as flat repositories have no +/// 'dists/' hierarchy. The suite name is read from the 'Release' file +/// unless overridden, and the package files are fetched relative to the URL +/// root, ignoring the 'Directory' field like apt does for flat repositories. +/// +/// Packages from external repositories are always downloaded from the +/// repository itself; the 'Vcs-Git' of the stanza is never used, as it may +/// point to an arbitrary source. +pub async fn lookup_repository( + package: &str, + version: Option<&str>, + repo_url: &str, + series: Option<&str>, + progress: ProgressCallback<'_>, +) -> Result> { + if let Some(cb) = progress { + cb( + &format!("Fetching repository info from {}...", repo_url), + "", + 0, + 0, + ); + } + + let resolved_series = if let Some(s) = series { + s.to_string() + } else { + get_flat_repo_series(repo_url).await? + }; + + if let Some(cb) = progress { + cb( + &format!("Resolving package info for {}...", package), + "", + 0, + 0, + ); + } + + let sources = get_flat_repo_sources(repo_url).await?; + let stanza = parse_sources(&sources, package, version)? + .ok_or_else(|| format!("Package '{package}' not found in repository {repo_url}"))?; + + // The distribution is only used for naming (e.g. git branch layout); + // fall back to a neutral name for suites unknown to distro-info. + let dist = crate::distro_info::get_dist_from_series(&resolved_series) + .await + .unwrap_or_else(|_| "external".to_string()); + + Ok(PackageInfo { + dist, + series: resolved_series, + stanza, + preferred_vcs: None, + archive_url: repo_url.trim_end_matches('/').to_string(), + }) +} + /// Lookup package information for a source package /// /// This function obtains package information either directly from a specific series @@ -435,7 +567,9 @@ async fn find_package( /// * `pocket` - Pocket to search in (e.g., "updates", "security", or "" for main) /// * `dist` - Optional distribution name (e.g., "ubuntu", "debian") /// * `base_url` - Optional base URL for the package archive (e.g., "https://ppa.launchpadcontent.net/user/ppa/ubuntu/") +/// * `repository` - Optional URL of an external flat repository suite (e.g., "https://pkg.noctalia.dev/deb/resolute/") /// * `progress` - Optional progress callback +#[allow(clippy::too_many_arguments)] pub async fn lookup( package: &str, version: Option<&str>, @@ -443,8 +577,15 @@ pub async fn lookup( pocket: &str, dist: Option<&str>, base_url: Option<&str>, + repository: Option<&str>, progress: ProgressCallback<'_>, ) -> Result> { + // External flat repository: the URL is the suite root, distribution + // resolution and pocket logic do not apply + if let Some(repo_url) = repository { + return lookup_repository(package, version, repo_url, series, progress).await; + } + // Resolve the distribution early so we can check Launchpad once let resolved_dist = dist.unwrap_or_else(|| // Use auto-detection to see if current distro is ubuntu, or fallback to debian by default @@ -582,6 +723,30 @@ Version: 1.0 assert!(none.is_none()); } + #[test] + fn test_parse_sources_compression_variants() { + use std::io::Write; + use xz2::write::XzEncoder; + + let data = "Package: hello +Version: 1.0 +Directory: pool/main/h/hello +"; + + // xz-compressed index (e.g. the 'Sources.xz' of flat repositories) + let mut encoder = XzEncoder::new(Vec::new(), 6); + encoder.write_all(data.as_bytes()).unwrap(); + let compressed = encoder.finish().unwrap(); + let info = parse_sources(&compressed, "hello", None).unwrap().unwrap(); + assert_eq!(info.version, "1.0"); + + // Uncompressed index + let info = parse_sources(data.as_bytes(), "hello", None) + .unwrap() + .unwrap(); + assert_eq!(info.version, "1.0"); + } + #[tokio::test] async fn test_find_package_fallback() { // python2.7 is in bullseye but not above diff --git a/src/pull.rs b/src/pull.rs index b8650e4..1e59081 100644 --- a/src/pull.rs +++ b/src/pull.rs @@ -153,13 +153,23 @@ fn copy_file_times(src: &Path, dest: &Path) -> Result<(), Box> { Ok(()) } +/// Result of extracting an archive +struct ExtractedArchive { + /// Paths of the extracted files + files: Vec, + /// The archive used './'-prefixed entries: its contents were extracted + /// directly into the destination directory, and there is no + /// 'package-version/' top-level directory to relocate + in_place: bool, +} + /// Helper function to extract tar archive with progress tracking fn extract_tar_archive( file_path: &Path, dest: &Path, progress: ProgressCallback<'_>, decoder_factory: F, -) -> Result, Box> +) -> Result> where D: std::io::Read, F: Fn(File) -> D, @@ -178,11 +188,28 @@ where let mut archive = Archive::new(decoder); let mut extracted_files = Vec::new(); + let mut in_place = false; for entry in archive.entries()? { let mut entry = entry?; let path = entry.path()?.to_path_buf(); - let dest_path = dest.join(&path); + + // Archives built with './'-prefixed entries (common in third-party + // repositories) target the destination directory itself, with no + // 'package-version/' wrapper; skip their root entry, and remember + // the layout for the caller + let relative = if path.to_string_lossy().starts_with("./") { + in_place = true; + path.strip_prefix("./").unwrap_or(&path).to_path_buf() + } else { + path + }; + + if relative.as_os_str().is_empty() { + continue; + } + + let dest_path = dest.join(&relative); // Create parent directories if needed if let Some(parent) = dest_path.parent() { @@ -201,14 +228,17 @@ where } } - Ok(extracted_files) + Ok(ExtractedArchive { + files: extracted_files, + in_place, + }) } fn extract_archive( path: &Path, dest: &Path, progress: ProgressCallback<'_>, -) -> Result, Box> { +) -> Result> { let filename = path.file_name().unwrap().to_string_lossy(); if filename.ends_with(".tar.gz") || filename.ends_with(".tgz") { @@ -465,47 +495,52 @@ async fn fetch_archive_sources( continue; } - // List root directories extracted and use the first one as the source directory - debug!("Root directories extracted:"); - let mut source_dir: Option = None; - for file in &extracted { - let path = Path::new(file); - // Check if this is a directory and is at the archive root level - // (i.e., the path relative to extract_dir has no parent components) - if let Ok(relative_path) = path.strip_prefix(&extract_dir) - && relative_path.components().count() == 1 - && path.is_dir() - { - debug!("- {}", relative_path.file_name().unwrap().to_string_lossy()); - // Use the first directory found as the source - if source_dir.is_none() { - source_dir = Some(path.to_path_buf()); - } - } - } - - // Use the extracted directory as the source, assuming there is only one - if let Some(src_dir) = source_dir { - let target_dir = package_dir.join(&info.stanza.package); - - if target_dir.exists() { - // Target exists, we need to merge contents - for sub_entry in std::fs::read_dir(&src_dir)? { - let sub_entry = sub_entry?; - let sub_path = sub_entry.path(); - let target_path = target_dir.join(sub_entry.file_name()); - if sub_path.is_dir() { - std::fs::create_dir_all(&target_path)?; - // Recursively copy directory contents - copy_dir_all(&sub_path, &target_path)?; - } else { - std::fs::copy(&sub_path, &target_path)?; - copy_file_times(&sub_path, &target_path)?; + // Archives with './'-prefixed entries are already laid out + // directly in the package directory; only 'package-version/' + // style ones need their contents relocated + if !extracted.in_place { + // List root directories extracted and use the first one as the source directory + debug!("Root directories extracted:"); + let mut source_dir: Option = None; + for file in &extracted.files { + let path = Path::new(file); + // Check if this is a directory and is at the archive root level + // (i.e., the path relative to extract_dir has no parent components) + if let Ok(relative_path) = path.strip_prefix(&extract_dir) + && relative_path.components().count() == 1 + && path.is_dir() + { + debug!("- {}", relative_path.file_name().unwrap().to_string_lossy()); + // Use the first directory found as the source + if source_dir.is_none() { + source_dir = Some(path.to_path_buf()); } } - std::fs::remove_dir_all(&src_dir)?; - } else { - std::fs::rename(&src_dir, &target_dir)?; + } + + // Use the extracted directory as the source, assuming there is only one + if let Some(src_dir) = source_dir { + let target_dir = package_dir.join(&info.stanza.package); + + if target_dir.exists() { + // Target exists, we need to merge contents + for sub_entry in std::fs::read_dir(&src_dir)? { + let sub_entry = sub_entry?; + let sub_path = sub_entry.path(); + let target_path = target_dir.join(sub_entry.file_name()); + if sub_path.is_dir() { + std::fs::create_dir_all(&target_path)?; + // Recursively copy directory contents + copy_dir_all(&sub_path, &target_path)?; + } else { + std::fs::copy(&sub_path, &target_path)?; + copy_file_times(&sub_path, &target_path)?; + } + } + std::fs::remove_dir_all(&src_dir)?; + } else { + std::fs::rename(&src_dir, &target_dir)?; + } } } } @@ -679,7 +714,7 @@ mod tests { let cwd = temp_dir.path(); // Main 'pull' command: the one we want to test - let info = crate::package_info::lookup(package, None, series, "", dist, None, None) + let info = crate::package_info::lookup(package, None, series, "", dist, None, None, None) .await .unwrap(); pull(&info, Some(cwd), None, archive.unwrap_or(false))