pull: add --repository to pull from external flat repositories

Add a --repository flag taking the full suite URL of an external flat
repository (e.g. https://pkg.noctalia.dev/deb/resolute/), i.e. one with
no dists/ hierarchy, like apt's exact-path suites ('Suites: resolute/').

The suite name is read from the root Release file (Codename/Suite), the
sources index is fetched from the repository root as Sources.xz/gz/plain,
and package files are resolved against the URL root, ignoring the stanza
Directory field like apt does. As with PPAs, the stanza Vcs-Git is never
used for external repositories, so the source always comes from the
repository itself.

Also make the sources index parser detect compression by magic bytes
(gz/xz/plain) instead of assuming gzip, and fix extraction of archives
with './'-prefixed entries, which previously aborted and are now
extracted in place instead of being relocated.

src/build/mod.rs

@@ -1492,6 +1492,7 @@ Provides: virtual-thing (= 2.0), plain-virtual
                 Some(dist),
                 None,
                 None,
+                None,
             ))
             .unwrap_or_else(|e| panic!("package lookup failed for {}: {}", package, e));
         rt.block_on(crate::pull::pull(

src/deb/mod.rs

@@ -383,7 +383,7 @@ mod tests {
 
         log::info!("Pulling package {} from {}...", package, series);
         let package_info =
-            crate::package_info::lookup(package, None, Some(series), "", dist, None, None)
+            crate::package_info::lookup(package, None, Some(series), "", dist, None, None, None)
                 .await
                 .expect("Cannot lookup package information");
         crate::pull::pull(&package_info, Some(cwd), None, true)

src/main.rs

@@ -48,6 +48,8 @@ fn main() {
                 .arg(arg!(-v --version <version> "Target package version").required(false))
                 .arg(arg!(--archive "Only use the archive to download package source, not git").required(false))
                 .arg(arg!(--ppa <ppa> "Download the package from a specific PPA").required(false))
+                .arg(arg!(--repository <url> "Download the package from an external flat repository, given as its full suite URL (e.g. https://pkg.noctalia.dev/deb/resolute/)").required(false)
+                    .conflicts_with("ppa"))
                 .arg(arg!(-p --pocket <pocket> "Target package distribution pocket (updates, security, proposed)").required(false))
                 .arg(arg!(<package> "Target package")),
         )
@@ -138,6 +140,9 @@ fn main() {
             let dist = sub_matches.get_one::<String>("dist").map(|s| s.as_str());
             let version = sub_matches.get_one::<String>("version").map(|s| s.as_str());
             let ppa = sub_matches.get_one::<String>("ppa").map(|s| s.as_str());
+            let repository = sub_matches
+                .get_one::<String>("repository")
+                .map(|s| s.as_str());
             let pocket = sub_matches
                 .get_one::<String>("pocket")
                 .map(|s| s.as_str())
@@ -166,6 +171,7 @@ fn main() {
                     pocket,
                     dist,
                     base_url.as_deref(),
+                    repository,
                     Some(&progress_callback),
                 )
                 .await?;

src/package_info.rs

@@ -2,6 +2,7 @@ use flate2::read::GzDecoder;
 use std::collections::HashMap;
 use std::error::Error;
 use std::io::Read;
+use xz2::read::XzDecoder;
 
 use crate::ProgressCallback;
 use crossterm::style::Stylize;
@@ -151,15 +152,29 @@ impl PackageInfo {
     }
 }
 
+/// Decompress a package index payload, detecting the compression format from
+/// its magic bytes: gzip ('1f 8b'), xz ('fd 7zXZ 00'), or plain text
+fn decompress_index(data: &[u8]) -> Result<String, Box<dyn Error>> {
+    let mut decoded: Box<dyn Read> = if data.starts_with(&[0x1f, 0x8b]) {
+        Box::new(GzDecoder::new(data))
+    } else if data.starts_with(&[0xfd, 0x37, b'z', b'X', b'Z', 0x00]) {
+        Box::new(XzDecoder::new(data))
+    } else {
+        return Ok(String::from_utf8(data.to_vec())?);
+    };
+
+    let mut s = String::new();
+    decoded.read_to_string(&mut s)?;
+    Ok(s)
+}
+
 struct DebianSources {
     splitted_sources: std::str::Split<'static, &'static str>,
 }
 impl DebianSources {
     fn new(data: &[u8]) -> Result<DebianSources, Box<dyn Error>> {
-        // Gz-decode 'Sources.gz' file into a string, and split it on stanzas
-        let mut d = GzDecoder::new(data);
-        let mut s = String::new();
-        d.read_to_string(&mut s)?;
+        // Decode the index into a string, and split it on stanzas
+        let s = decompress_index(data)?;
 
         // Convert the string to a static lifetime by leaking it
         let static_str = Box::leak(s.into_boxed_str());
@@ -423,6 +438,123 @@ async fn find_package(
     Err(format!("Package '{}' not found.", package_name).into())
 }
 
+/// Fetch the 'Release' file at the root of a flat repository, and return its suite name
+///
+/// Flat repositories (e.g. 'https://pkg.example.org/deb/resolute/') serve the
+/// Release file at their root, without any 'dists/' hierarchy. The suite is
+/// read from the 'Codename' field, falling back to 'Suite'.
+async fn get_flat_repo_series(base_url: &str) -> Result<String, Box<dyn Error>> {
+    let url = format!("{}/Release", base_url.trim_end_matches('/'));
+    let response = reqwest::get(&url).await?;
+    if !response.status().is_success() {
+        return Err(format!(
+            "No Release file at '{}' (HTTP {}) - is '{}' the suite URL of a flat repository? \
+             External repositories must be passed as their full suite URL.",
+            url,
+            response.status(),
+            base_url
+        )
+        .into());
+    }
+    let content = response.text().await?;
+
+    for field in ["Codename", "Suite"] {
+        let prefix = format!("{field}:");
+        if let Some(line) = content.lines().find(|l| l.starts_with(&prefix)) {
+            return Ok(line[prefix.len()..].trim().to_string());
+        }
+    }
+
+    Err(format!("No Codename or Suite field in Release file at '{url}'").into())
+}
+
+/// Fetch the sources index of a flat repository
+///
+/// Flat repositories are free to serve any compressed variant of the index
+/// (or an uncompressed one), so try the usual candidates in turn.
+async fn get_flat_repo_sources(base_url: &str) -> Result<Vec<u8>, Box<dyn Error>> {
+    let base = base_url.trim_end_matches('/');
+    let mut errors = Vec::new();
+    for name in ["Sources.xz", "Sources.gz", "Sources"] {
+        let url = format!("{base}/{name}");
+        match reqwest::get(&url).await {
+            Ok(response) if response.status().is_success() => {
+                return Ok(response.bytes().await?.to_vec());
+            }
+            Ok(response) => errors.push(format!("{}: HTTP {}", url, response.status())),
+            Err(e) => errors.push(format!("{}: {}", url, e)),
+        }
+    }
+
+    Err(format!(
+        "Could not fetch the sources index from '{}': {}",
+        base,
+        errors.join(", ")
+    )
+    .into())
+}
+
+/// Lookup package information in an external flat repository
+///
+/// The URL points at the repository suite directory itself (e.g.
+/// 'https://pkg.noctalia.dev/deb/resolute/'), as flat repositories have no
+/// 'dists/<suite>' hierarchy. The suite name is read from the 'Release' file
+/// unless overridden, and the package files are fetched relative to the URL
+/// root, ignoring the 'Directory' field like apt does for flat repositories.
+///
+/// Packages from external repositories are always downloaded from the
+/// repository itself; the 'Vcs-Git' of the stanza is never used, as it may
+/// point to an arbitrary source.
+pub async fn lookup_repository(
+    package: &str,
+    version: Option<&str>,
+    repo_url: &str,
+    series: Option<&str>,
+    progress: ProgressCallback<'_>,
+) -> Result<PackageInfo, Box<dyn Error>> {
+    if let Some(cb) = progress {
+        cb(
+            &format!("Fetching repository info from {}...", repo_url),
+            "",
+            0,
+            0,
+        );
+    }
+
+    let resolved_series = if let Some(s) = series {
+        s.to_string()
+    } else {
+        get_flat_repo_series(repo_url).await?
+    };
+
+    if let Some(cb) = progress {
+        cb(
+            &format!("Resolving package info for {}...", package),
+            "",
+            0,
+            0,
+        );
+    }
+
+    let sources = get_flat_repo_sources(repo_url).await?;
+    let stanza = parse_sources(&sources, package, version)?
+        .ok_or_else(|| format!("Package '{package}' not found in repository {repo_url}"))?;
+
+    // The distribution is only used for naming (e.g. git branch layout);
+    // fall back to a neutral name for suites unknown to distro-info.
+    let dist = crate::distro_info::get_dist_from_series(&resolved_series)
+        .await
+        .unwrap_or_else(|_| "external".to_string());
+
+    Ok(PackageInfo {
+        dist,
+        series: resolved_series,
+        stanza,
+        preferred_vcs: None,
+        archive_url: repo_url.trim_end_matches('/').to_string(),
+    })
+}
+
 /// Lookup package information for a source package
 ///
 /// This function obtains package information either directly from a specific series
@@ -435,7 +567,9 @@ async fn find_package(
 /// * `pocket` - Pocket to search in (e.g., "updates", "security", or "" for main)
 /// * `dist` - Optional distribution name (e.g., "ubuntu", "debian")
 /// * `base_url` - Optional base URL for the package archive (e.g., "https://ppa.launchpadcontent.net/user/ppa/ubuntu/")
+/// * `repository` - Optional URL of an external flat repository suite (e.g., "https://pkg.noctalia.dev/deb/resolute/")
 /// * `progress` - Optional progress callback
+#[allow(clippy::too_many_arguments)]
 pub async fn lookup(
     package: &str,
     version: Option<&str>,
@@ -443,8 +577,15 @@ pub async fn lookup(
     pocket: &str,
     dist: Option<&str>,
     base_url: Option<&str>,
+    repository: Option<&str>,
     progress: ProgressCallback<'_>,
 ) -> Result<PackageInfo, Box<dyn Error>> {
+    // External flat repository: the URL is the suite root, distribution
+    // resolution and pocket logic do not apply
+    if let Some(repo_url) = repository {
+        return lookup_repository(package, version, repo_url, series, progress).await;
+    }
+
     // Resolve the distribution early so we can check Launchpad once
     let resolved_dist = dist.unwrap_or_else(||
         // Use auto-detection to see if current distro is ubuntu, or fallback to debian by default
@@ -582,6 +723,30 @@ Version: 1.0
         assert!(none.is_none());
     }
 
+    #[test]
+    fn test_parse_sources_compression_variants() {
+        use std::io::Write;
+        use xz2::write::XzEncoder;
+
+        let data = "Package: hello
+Version: 1.0
+Directory: pool/main/h/hello
+";
+
+        // xz-compressed index (e.g. the 'Sources.xz' of flat repositories)
+        let mut encoder = XzEncoder::new(Vec::new(), 6);
+        encoder.write_all(data.as_bytes()).unwrap();
+        let compressed = encoder.finish().unwrap();
+        let info = parse_sources(&compressed, "hello", None).unwrap().unwrap();
+        assert_eq!(info.version, "1.0");
+
+        // Uncompressed index
+        let info = parse_sources(data.as_bytes(), "hello", None)
+            .unwrap()
+            .unwrap();
+        assert_eq!(info.version, "1.0");
+    }
+
     #[tokio::test]
     async fn test_find_package_fallback() {
         // python2.7 is in bullseye but not above

src/pull.rs

@@ -153,13 +153,23 @@ fn copy_file_times(src: &Path, dest: &Path) -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
+/// Result of extracting an archive
+struct ExtractedArchive {
+    /// Paths of the extracted files
+    files: Vec<String>,
+    /// The archive used './'-prefixed entries: its contents were extracted
+    /// directly into the destination directory, and there is no
+    /// 'package-version/' top-level directory to relocate
+    in_place: bool,
+}
+
 /// Helper function to extract tar archive with progress tracking
 fn extract_tar_archive<D, F>(
     file_path: &Path,
     dest: &Path,
     progress: ProgressCallback<'_>,
     decoder_factory: F,
-) -> Result<Vec<String>, Box<dyn Error>>
+) -> Result<ExtractedArchive, Box<dyn Error>>
 where
     D: std::io::Read,
     F: Fn(File) -> D,
@@ -178,11 +188,28 @@ where
     let mut archive = Archive::new(decoder);
 
     let mut extracted_files = Vec::new();
+    let mut in_place = false;
 
     for entry in archive.entries()? {
         let mut entry = entry?;
         let path = entry.path()?.to_path_buf();
-        let dest_path = dest.join(&path);
+
+        // Archives built with './'-prefixed entries (common in third-party
+        // repositories) target the destination directory itself, with no
+        // 'package-version/' wrapper; skip their root entry, and remember
+        // the layout for the caller
+        let relative = if path.to_string_lossy().starts_with("./") {
+            in_place = true;
+            path.strip_prefix("./").unwrap_or(&path).to_path_buf()
+        } else {
+            path
+        };
+
+        if relative.as_os_str().is_empty() {
+            continue;
+        }
+
+        let dest_path = dest.join(&relative);
 
         // Create parent directories if needed
         if let Some(parent) = dest_path.parent() {
@@ -201,14 +228,17 @@ where
         }
     }
 
-    Ok(extracted_files)
+    Ok(ExtractedArchive {
+        files: extracted_files,
+        in_place,
+    })
 }
 
 fn extract_archive(
     path: &Path,
     dest: &Path,
     progress: ProgressCallback<'_>,
-) -> Result<Vec<String>, Box<dyn Error>> {
+) -> Result<ExtractedArchive, Box<dyn Error>> {
     let filename = path.file_name().unwrap().to_string_lossy();
 
     if filename.ends_with(".tar.gz") || filename.ends_with(".tgz") {
@@ -465,47 +495,52 @@ async fn fetch_archive_sources(
                 continue;
             }
 
-            // List root directories extracted and use the first one as the source directory
-            debug!("Root directories extracted:");
-            let mut source_dir: Option<PathBuf> = None;
-            for file in &extracted {
-                let path = Path::new(file);
-                // Check if this is a directory and is at the archive root level
-                // (i.e., the path relative to extract_dir has no parent components)
-                if let Ok(relative_path) = path.strip_prefix(&extract_dir)
-                    && relative_path.components().count() == 1
-                    && path.is_dir()
-                {
-                    debug!("- {}", relative_path.file_name().unwrap().to_string_lossy());
-                    // Use the first directory found as the source
-                    if source_dir.is_none() {
-                        source_dir = Some(path.to_path_buf());
-                    }
-                }
-            }
-
-            // Use the extracted directory as the source, assuming there is only one
-            if let Some(src_dir) = source_dir {
-                let target_dir = package_dir.join(&info.stanza.package);
-
-                if target_dir.exists() {
-                    // Target exists, we need to merge contents
-                    for sub_entry in std::fs::read_dir(&src_dir)? {
-                        let sub_entry = sub_entry?;
-                        let sub_path = sub_entry.path();
-                        let target_path = target_dir.join(sub_entry.file_name());
-                        if sub_path.is_dir() {
-                            std::fs::create_dir_all(&target_path)?;
-                            // Recursively copy directory contents
-                            copy_dir_all(&sub_path, &target_path)?;
-                        } else {
-                            std::fs::copy(&sub_path, &target_path)?;
-                            copy_file_times(&sub_path, &target_path)?;
+            // Archives with './'-prefixed entries are already laid out
+            // directly in the package directory; only 'package-version/'
+            // style ones need their contents relocated
+            if !extracted.in_place {
+                // List root directories extracted and use the first one as the source directory
+                debug!("Root directories extracted:");
+                let mut source_dir: Option<PathBuf> = None;
+                for file in &extracted.files {
+                    let path = Path::new(file);
+                    // Check if this is a directory and is at the archive root level
+                    // (i.e., the path relative to extract_dir has no parent components)
+                    if let Ok(relative_path) = path.strip_prefix(&extract_dir)
+                        && relative_path.components().count() == 1
+                        && path.is_dir()
+                    {
+                        debug!("- {}", relative_path.file_name().unwrap().to_string_lossy());
+                        // Use the first directory found as the source
+                        if source_dir.is_none() {
+                            source_dir = Some(path.to_path_buf());
                         }
                     }
-                    std::fs::remove_dir_all(&src_dir)?;
-                } else {
-                    std::fs::rename(&src_dir, &target_dir)?;
+                }
+
+                // Use the extracted directory as the source, assuming there is only one
+                if let Some(src_dir) = source_dir {
+                    let target_dir = package_dir.join(&info.stanza.package);
+
+                    if target_dir.exists() {
+                        // Target exists, we need to merge contents
+                        for sub_entry in std::fs::read_dir(&src_dir)? {
+                            let sub_entry = sub_entry?;
+                            let sub_path = sub_entry.path();
+                            let target_path = target_dir.join(sub_entry.file_name());
+                            if sub_path.is_dir() {
+                                std::fs::create_dir_all(&target_path)?;
+                                // Recursively copy directory contents
+                                copy_dir_all(&sub_path, &target_path)?;
+                            } else {
+                                std::fs::copy(&sub_path, &target_path)?;
+                                copy_file_times(&sub_path, &target_path)?;
+                            }
+                        }
+                        std::fs::remove_dir_all(&src_dir)?;
+                    } else {
+                        std::fs::rename(&src_dir, &target_dir)?;
+                    }
                 }
             }
         }
@@ -679,7 +714,7 @@ mod tests {
         let cwd = temp_dir.path();
 
         // Main 'pull' command: the one we want to test
-        let info = crate::package_info::lookup(package, None, series, "", dist, None, None)
+        let info = crate::package_info::lookup(package, None, series, "", dist, None, None, None)
             .await
             .unwrap();
         pull(&info, Some(cwd), None, archive.unwrap_or(false))