pull: allow pulling from ppa

.github/workflows/ci.yml

@@ -49,3 +49,17 @@ jobs:
       env:
         RUST_LOG: debug
       run: timeout 30m cargo test -- --nocapture
+
+  snap:
+    needs: build
+    runs-on: ubuntu-latest
+    outputs:
+      snap-file: ${{ steps.build-snap.outputs.snap }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: snapcore/action-build@v1
+        id: build-snap
+      - uses: actions/upload-artifact@v3
+        with:
+          name: snap
+          path: ${{ steps.build-snap.outputs.snap }}

README.md

@@ -82,7 +82,7 @@ Missing features:
 - [ ] `pkh pull`
   - [x] Obtain package sources from git
   - [x] Obtain package sources from the archive (fallback)
-  - [ ] Obtain package source from PPA (--ppa)
+  - [x] Obtain package source from PPA (--ppa)
   - [ ] Obtain a specific version of the package
   - [x] Fetch the correct git branch for series on Ubuntu
   - [ ] Try to fetch the correct git branch for series on Debian, or fallback to the archive

snap/snapcraft.yaml

@@ -0,0 +1,42 @@
+name: pkh
+base: core24
+summary: pkh is a packaging helper for Debian/Ubuntu packages
+description: |
+  pkh aims at wrapping the different debian tools and workflows 
+  into one tool, that would have the same interface for everything, 
+  while being smarter at integrating all workflows.
+adopt-info: pkh-part
+
+confinement: devmode
+
+apps:
+    pkh:
+      command: bin/pkh
+
+parts:
+  pkh-part:
+    plugin: rust
+    source: .
+    override-pull: |
+      craftctl default
+      craftctl set version=$(git rev-parse --short=11 HEAD)
+      craftctl set grade="devel"
+    build-packages:
+      - pkg-config
+      - libssl-dev
+      - libgpg-error-dev
+      - libgpgme-dev
+    stage-packages:
+      - libgpgme11t64
+      - git
+      - curl
+      - pristine-tar
+      - sbuild
+      - mmdebstrap
+      - util-linux
+      - dpkg-dev
+    stage:
+    - -usr/lib/x86_64-linux-gnu/libicuio.so.74.2
+    - -usr/lib/x86_64-linux-gnu/libicutest.so.74.2
+    - -usr/lib/x86_64-linux-gnu/libicutu.so.74.2
+    - -usr/lib/x86_64-linux-gnu/libicui18n.so.74.2

src/apt/keyring.rs

@@ -26,8 +26,12 @@ pub async fn download_trust_keyring(
         ctx.command("mkdir").arg("-p").arg(trusted_gpg_d).status()?;
     }
 
-    // Generate a filename for the keyring
-    let filename = format!("pkh-{}.gpg", series);
+    // Extract the original filename from the keyring URL
+    let filename = keyring_url
+        .split('/')
+        .next_back()
+        .unwrap_or("pkh-{}.gpg")
+        .replace("{}", series);
     let keyring_path = format!("{}/{}", trusted_gpg_d, filename);
 
     // Download the keyring directly to the final location using curl

src/context/unshare.rs

@@ -208,7 +208,7 @@ impl UnshareDriver {
         }
 
         cmd.arg("--").arg("bash").arg("-c").arg(format!(
-            "mount -t proc proc /proc; mount -t devpts devpts /dev/pts; mount --bind /dev/pts/ptmx /dev/ptmx; {} {}",
+            "mount -t proc proc /proc; mkdir /dev/pts; mount -t devpts devpts /dev/pts; touch /dev/ptmx; mount --bind /dev/pts/ptmx /dev/ptmx; {} {}",
             program,
             args.join(" ")
         ));

src/deb/local.rs

@@ -24,6 +24,26 @@ pub fn build(
 
     let ctx = context::current();
 
+    // Parallel building: find local number of cores, and use that
+    let num_cores = ctx
+        .command("nproc")
+        .output()
+        .map(|output| {
+            if output.status.success() {
+                String::from_utf8_lossy(&output.stdout)
+                    .trim()
+                    .parse::<usize>()
+                    .unwrap_or(1)
+            } else {
+                1 // Default to 1 if nproc fails
+            }
+        })
+        .unwrap_or(1); // Default to 1 if we can't execute the command
+    env.insert(
+        "DEB_BUILD_OPTIONS".to_string(),
+        format!("parallel={}", num_cores),
+    );
+
     if cross {
         log::debug!("Setting up environment for local cross build...");
         cross::setup_environment(&mut env, arch)?;

src/deb/mod.rs

@@ -102,7 +102,7 @@ pub(crate) fn find_package_directory(
 
     // Try package/package pattern first
     let package_dir = parent_dir.join(package).join(package);
-    if ctx.exists(&package_dir)? {
+    if ctx.exists(&package_dir)? && ctx.exists(&package_dir.join("debian"))? {
         return Ok(package_dir);
     }
 
@@ -117,19 +117,19 @@ pub(crate) fn find_package_directory(
     let package_dir = parent_dir
         .join(package)
         .join(format!("{}-{}", package, origversion));
-    if ctx.exists(&package_dir)? {
+    if ctx.exists(&package_dir)? && ctx.exists(&package_dir.join("debian"))? {
         return Ok(package_dir);
     }
 
     // Try 'package' only
     let package_dir = parent_dir.join(package);
-    if ctx.exists(&package_dir)? {
+    if ctx.exists(&package_dir)? && ctx.exists(&package_dir.join("debian"))? {
         return Ok(package_dir);
     }
 
     // Try package-origversion only
     let package_dir = parent_dir.join(format!("{}-{}", package, origversion));
-    if ctx.exists(&package_dir)? {
+    if ctx.exists(&package_dir)? && ctx.exists(&package_dir.join("debian"))? {
         return Ok(package_dir);
     }
 
@@ -212,9 +212,10 @@ mod tests {
         log::debug!("Created temporary directory: {}", cwd.display());
 
         log::info!("Pulling package {} from {}...", package, series);
-        let package_info = crate::package_info::lookup(package, None, Some(series), "", dist, None)
-            .await
-            .expect("Cannot lookup package information");
+        let package_info =
+            crate::package_info::lookup(package, None, Some(series), "", dist, None, None)
+                .await
+                .expect("Cannot lookup package information");
         crate::pull::pull(&package_info, Some(cwd), None, true)
             .await
             .expect("Cannot pull package");
@@ -265,6 +266,7 @@ mod tests {
         test_build_end_to_end("hello", "noble", None, None, false).await;
     }
 
+    /// This ensures that we can cross-build packages
     #[tokio::test]
     #[test_log::test]
     #[cfg(target_arch = "x86_64")]
@@ -273,10 +275,25 @@ mod tests {
         test_build_end_to_end("hello", "noble", None, Some("riscv64"), true).await;
     }
 
+    /// This ensures that we can build packages from sid, even on older
+    /// releases. It can sometimes be difficult with mmdebstrap issues
+    /// for example.
+    #[tokio::test]
+    #[test_log::test]
+    #[serial]
+    async fn test_deb_hello_debian_sid_end_to_end() {
+        test_build_end_to_end("hello", "sid", None, None, false).await;
+    }
+
     /// This is a specific test case for the latest gcc package on Debian
     /// The GCC package is complex and hard to build, with specific stages
     /// and system-bound scripts. Building it requires specific things that
     /// we want to ensure are not broken.
+    /// NOTE: Ideally, we want to run this in CI, but it takes more than 20h
+    /// to fully build the gcc-15 package on an amd64 builder, which is too
+    /// much time.
+    #[ignore]
+    #[cfg(target_arch = "x86_64")]
     #[tokio::test]
     #[test_log::test]
     #[serial]

src/distro_info.rs

@@ -4,6 +4,23 @@ use serde::Deserialize;
 use std::error::Error;
 use std::path::Path;
 
+#[derive(Debug, Clone)]
+/// Information about a specific distribution series
+pub struct SeriesInformation {
+    /// Distribution series
+    pub series: String,
+    /// Codename, i.e. full name of series
+    pub codename: String,
+    /// Series version as numbers
+    pub version: Option<String>,
+    /// Series creation date
+    pub created: NaiveDate,
+    /// Series release date
+    pub release: Option<NaiveDate>,
+    /// Series end-of-life date
+    pub eol: Option<NaiveDate>,
+}
+
 #[derive(Debug, Deserialize)]
 struct SeriesInfo {
     local: String,
@@ -28,7 +45,7 @@ lazy_static! {
     static ref DATA: Data = serde_yaml::from_str(DATA_YAML).unwrap();
 }
 
-fn parse_series_csv(content: &str) -> Result<Vec<String>, Box<dyn Error>> {
+fn parse_series_csv(content: &str) -> Result<Vec<SeriesInformation>, Box<dyn Error>> {
     let mut rdr = csv::ReaderBuilder::new()
         .flexible(true)
         .from_reader(content.as_bytes());
@@ -38,29 +55,63 @@ fn parse_series_csv(content: &str) -> Result<Vec<String>, Box<dyn Error>> {
         .iter()
         .position(|h| h == "series")
         .ok_or("Column 'series' not found")?;
+    let codename_idx = headers
+        .iter()
+        .position(|h| h == "codename")
+        .ok_or("Column 'codename' not found")?;
+    let version_idx = headers
+        .iter()
+        .position(|h| h == "version")
+        .ok_or("Column 'version' not found")?;
     let created_idx = headers
         .iter()
         .position(|h| h == "created")
         .ok_or("Column 'created' not found")?;
+    let release_idx = headers
+        .iter()
+        .position(|h| h == "release")
+        .ok_or("Column 'release' not found")?;
+    let eol_idx = headers
+        .iter()
+        .position(|h| h == "eol")
+        .ok_or("Column 'eol' not found")?;
+
+    let mut series_info_list = Vec::new();
 
-    let mut entries = Vec::new();
     for result in rdr.records() {
         let record = result?;
-        if let (Some(s), Some(c)) = (record.get(series_idx), record.get(created_idx))
-            && let Ok(date) = NaiveDate::parse_from_str(c, "%Y-%m-%d")
-        {
-            entries.push((s.to_string(), date));
-        }
+        let series = record.get(series_idx).unwrap().to_string();
+        let codename = record.get(codename_idx).unwrap().to_string();
+        let version = record.get(version_idx).map(|s| s.to_string());
+        let created = record
+            .get(created_idx)
+            .map(|date_str| NaiveDate::parse_from_str(date_str, "%Y-%m-%d").unwrap())
+            .unwrap();
+        let release = record
+            .get(release_idx)
+            .map(|date_str| NaiveDate::parse_from_str(date_str, "%Y-%m-%d").unwrap());
+        let eol = record
+            .get(eol_idx)
+            .map(|date_str| NaiveDate::parse_from_str(date_str, "%Y-%m-%d").unwrap());
+
+        series_info_list.push(SeriesInformation {
+            series,
+            codename,
+            version,
+            created,
+            release,
+            eol,
+        });
     }
 
-    // Sort by date descending (newest first)
-    entries.sort_by(|a, b| b.1.cmp(&a.1));
+    // Revert to sort by most recent
+    series_info_list.reverse();
 
-    Ok(entries.into_iter().map(|(s, _)| s).collect())
+    Ok(series_info_list)
 }
 
-/// Get time-ordered list of series for a distribution, development series first
-pub async fn get_ordered_series(dist: &str) -> Result<Vec<String>, Box<dyn Error>> {
+/// Get time-ordered list of series information for a distribution, development series first
+pub async fn get_ordered_series(dist: &str) -> Result<Vec<SeriesInformation>, Box<dyn Error>> {
     let series_info = &DATA.dist.get(dist).unwrap().series;
     let content = if Path::new(series_info.local.as_str()).exists() {
         std::fs::read_to_string(format!("/usr/share/distro-info/{dist}.csv"))?
@@ -71,22 +122,48 @@ pub async fn get_ordered_series(dist: &str) -> Result<Vec<String>, Box<dyn Error
             .await?
     };
 
-    let mut series = parse_series_csv(&content)?;
+    let series_info_list = parse_series_csv(&content)?;
+    Ok(series_info_list)
+}
 
-    // For Debian, ensure 'sid' is first if it's not
-    // We want to try 'sid' (unstable) first for Debian.
-    if dist == "debian" {
-        series.retain(|s| s != "sid");
-        series.insert(0, "sid".to_string());
+/// Get time-ordered list of series names for a distribution, development series first
+pub async fn get_ordered_series_name(dist: &str) -> Result<Vec<String>, Box<dyn Error>> {
+    let series = get_ordered_series(dist).await?;
+    Ok(series.iter().map(|info| info.series.clone()).collect())
+}
+
+/// Get the latest released series for a dist (excluding future releases and special cases like sid)
+pub async fn get_latest_released_series(dist: &str) -> Result<String, Box<dyn Error>> {
+    let series_info_list = get_ordered_series(dist).await?;
+
+    let today = chrono::Local::now().date_naive();
+    let mut released_series = Vec::new();
+
+    for series_info in series_info_list {
+        // Skip 'sid' and series without release dates or with future release dates
+        if series_info.series != "sid"
+            && series_info.series != "experimental"
+            && series_info.release.is_some()
+            && series_info.release.unwrap() <= today
+        {
+            released_series.push(series_info);
+        }
     }
 
-    Ok(series)
+    // Sort by release date descending (newest first)
+    released_series.sort_by(|a, b| b.release.cmp(&a.release));
+
+    if let Some(latest) = released_series.first() {
+        Ok(latest.series.clone())
+    } else {
+        Err("No released series found".into())
+    }
 }
 
 /// Obtain the distribution (eg. debian, ubuntu) from a distribution series (eg. noble, bookworm)
 pub async fn get_dist_from_series(series: &str) -> Result<String, Box<dyn Error>> {
     for dist in DATA.dist.keys() {
-        if get_ordered_series(dist)
+        if get_ordered_series_name(dist)
             .await?
             .contains(&series.to_string())
         {
@@ -135,11 +212,21 @@ pub async fn get_keyring_url(series: &str) -> Result<String, Box<dyn Error>> {
 
     // For Debian, we need the series number to form the keyring URL
     if dist == "debian" {
-        let series_num = get_debian_series_number(series).await?.unwrap();
-        // Replace {series_num} placeholder with the actual series number
-        Ok(dist_data
-            .archive_keyring
-            .replace("{series_num}", &series_num))
+        // Special case for 'sid' - use the latest released version
+        if series == "sid" || series == "experimental" {
+            let latest_released = get_latest_released_series("debian").await?;
+            let series_num = get_debian_series_number(&latest_released).await?.unwrap();
+            // Replace {series_num} placeholder with the latest released series number
+            Ok(dist_data
+                .archive_keyring
+                .replace("{series_num}", &series_num))
+        } else {
+            let series_num = get_debian_series_number(series).await?.unwrap();
+            // Replace {series_num} placeholder with the actual series number
+            Ok(dist_data
+                .archive_keyring
+                .replace("{series_num}", &series_num))
+        }
     } else {
         // For other distributions like Ubuntu, use the keyring directly
         Ok(dist_data.archive_keyring.clone())
@@ -225,14 +312,14 @@ mod tests {
 
     #[tokio::test]
     async fn test_get_debian_series() {
-        let series = get_ordered_series("debian").await.unwrap();
+        let series = get_ordered_series_name("debian").await.unwrap();
         assert!(series.contains(&"sid".to_string()));
         assert!(series.contains(&"bookworm".to_string()));
     }
 
     #[tokio::test]
     async fn test_get_ubuntu_series() {
-        let series = get_ordered_series("ubuntu").await.unwrap();
+        let series = get_ordered_series_name("ubuntu").await.unwrap();
         assert!(series.contains(&"noble".to_string()));
         assert!(series.contains(&"jammy".to_string()));
     }
@@ -258,4 +345,29 @@ mod tests {
         let unknown_number = get_debian_series_number("unknown").await.unwrap();
         assert!(unknown_number.is_none());
     }
+
+    #[tokio::test]
+    async fn test_get_keyring_url_sid() {
+        // Test that 'sid' uses the latest released version for keyring URL
+        let sid_keyring = get_keyring_url("sid").await.unwrap();
+        let latest_released = get_latest_released_series("debian").await.unwrap();
+        let latest_keyring = get_keyring_url(&latest_released).await.unwrap();
+
+        // The keyring URL for 'sid' should be the same as the latest released version
+        assert_eq!(sid_keyring, latest_keyring);
+    }
+
+    #[tokio::test]
+    async fn test_get_latest_released_debian_series() {
+        // Test that we get a valid released series
+        let latest_released = get_latest_released_series("debian").await.unwrap();
+
+        // Should not be 'sid' or 'experimental'
+        assert_ne!(latest_released, "sid");
+        assert_ne!(latest_released, "experimental");
+
+        // Should have a version number
+        let version = get_debian_series_number(&latest_released).await.unwrap();
+        assert!(version.is_some());
+    }
 }

src/main.rs

@@ -94,14 +94,22 @@ fn main() {
             let series = sub_matches.get_one::<String>("series").map(|s| s.as_str());
             let dist = sub_matches.get_one::<String>("dist").map(|s| s.as_str());
             let version = sub_matches.get_one::<String>("version").map(|s| s.as_str());
-            let _ppa = sub_matches
-                .get_one::<String>("ppa")
-                .map(|s| s.as_str())
-                .unwrap_or("");
+            let ppa = sub_matches.get_one::<String>("ppa").map(|s| s.as_str());
             let archive = sub_matches.get_one::<bool>("archive").unwrap_or(&false);
 
             let (pb, progress_callback) = ui::create_progress_bar(&multi);
 
+            // Convert PPA to base URL if provided
+            let base_url = ppa.and_then(|ppa_str| {
+                // PPA format: user/ppa_name
+                let parts: Vec<&str> = ppa_str.split('/').collect();
+                if parts.len() == 2 {
+                    Some(pkh::package_info::ppa_to_base_url(parts[0], parts[1]))
+                } else {
+                    None
+                }
+            });
+
             // Since pull is async, we need to block on it
             if let Err(e) = rt.block_on(async {
                 let package_info = pkh::package_info::lookup(
@@ -110,6 +118,7 @@ fn main() {
                     series,
                     "",
                     dist,
+                    base_url.as_deref(),
                     Some(&progress_callback),
                 )
                 .await?;

src/package_info.rs

@@ -6,6 +6,18 @@ use std::io::Read;
 use crate::ProgressCallback;
 use log::{debug, warn};
 
+/// Convert a PPA specification to a base URL
+///
+/// # Arguments
+/// * user: user for the PPA
+/// * name: name of the PPA
+///
+/// # Returns
+/// * The base URL for the PPA (e.g., "https://ppa.launchpadcontent.net/user/ppa_name/ubuntu/")
+pub fn ppa_to_base_url(user: &str, name: &str) -> String {
+    format!("https://ppa.launchpadcontent.net/{}/{}/ubuntu", user, name)
+}
+
 async fn check_launchpad_repo(package: &str) -> Result<Option<String>, Box<dyn Error>> {
     let url = format!("https://git.launchpad.net/ubuntu/+source/{}", package);
     let client = reqwest::Client::builder()
@@ -178,6 +190,7 @@ async fn get(
     series: &str,
     pocket: &str,
     version: Option<&str>,
+    base_url: Option<&str>,
 ) -> Result<PackageInfo, Box<dyn Error>> {
     let dist = crate::distro_info::get_dist_from_series(series).await?;
 
@@ -191,7 +204,16 @@ async fn get(
         preferred_vcs = Some(lp_url);
     }
 
-    let base_url = crate::distro_info::get_base_url(&dist);
+    // Determine the base URL to use (either provided PPA URL or default archive)
+    let distro_base_url = crate::distro_info::get_base_url(&dist);
+    let base_url = if let Some(ppa_url) = base_url {
+        ppa_url.to_string()
+    } else {
+        distro_base_url.clone()
+    };
+
+    // If using a custom base URL (PPA), disable VCS lookup to force archive download
+    let from_ppa = base_url != distro_base_url;
 
     let components = crate::distro_info::get_components(&base_url, series, pocket).await?;
     debug!("Found components: {:?}", components);
@@ -228,6 +250,11 @@ async fn get(
                 preferred_vcs = Some(vcs.clone());
             }
 
+            // If downloading from PPA, make sure we don't use a VCS
+            if from_ppa {
+                preferred_vcs = None;
+            }
+
             let archive_url = format!("{base_url}/{0}", stanza.directory);
             return Ok(PackageInfo {
                 dist,
@@ -252,9 +279,10 @@ async fn find_package(
     dist: &str,
     pocket: &str,
     version: Option<&str>,
+    base_url: Option<&str>,
     progress: ProgressCallback<'_>,
 ) -> Result<PackageInfo, Box<dyn Error>> {
-    let series_list = crate::distro_info::get_ordered_series(dist).await?;
+    let series_list = crate::distro_info::get_ordered_series_name(dist).await?;
 
     for (i, series) in series_list.iter().enumerate() {
         if let Some(cb) = progress {
@@ -268,7 +296,7 @@ async fn find_package(
         };
 
         for p in pockets {
-            match get(package_name, series, &p, version).await {
+            match get(package_name, series, &p, version, base_url).await {
                 Ok(info) => {
                     if i > 0 {
                         warn!(
@@ -297,12 +325,22 @@ async fn find_package(
 ///
 /// This function obtains package information either directly from a specific series
 /// or by searching across all series in a distribution.
+///
+/// # Arguments
+/// * `package` - The name of the package to look up
+/// * `version` - Optional specific version to look for
+/// * `series` - Optional distribution series (e.g., "noble", "bookworm")
+/// * `pocket` - Pocket to search in (e.g., "updates", "security", or "" for main)
+/// * `dist` - Optional distribution name (e.g., "ubuntu", "debian")
+/// * `base_url` - Optional base URL for the package archive (e.g., "https://ppa.launchpadcontent.net/user/ppa/ubuntu/")
+/// * `progress` - Optional progress callback
 pub async fn lookup(
     package: &str,
     version: Option<&str>,
     series: Option<&str>,
     pocket: &str,
     dist: Option<&str>,
+    base_url: Option<&str>,
     progress: ProgressCallback<'_>,
 ) -> Result<PackageInfo, Box<dyn Error>> {
     // Obtain the package information, either directly in a series or with a search in all series
@@ -317,7 +355,7 @@ pub async fn lookup(
         }
 
         // Get the package information from that series and pocket
-        get(package, s, pocket, version).await?
+        get(package, s, pocket, version, base_url).await?
     } else {
         let dist = dist.unwrap_or_else(||
             // Use auto-detection to see if current distro is ubuntu, or fallback to debian by default
@@ -331,7 +369,11 @@ pub async fn lookup(
 
         if let Some(cb) = progress {
             cb(
-                &format!("Searching for package {} in {}...", package, dist),
+                &format!(
+                    "Searching for package {} in {}...",
+                    package,
+                    if base_url.is_none() { dist } else { "ppa" }
+                ),
                 "",
                 0,
                 0,
@@ -339,7 +381,7 @@ pub async fn lookup(
         }
 
         // Try to find the package in all series from that dist
-        find_package(package, dist, pocket, version, progress).await?
+        find_package(package, dist, pocket, version, base_url, progress).await?
     };
 
     Ok(package_info)
@@ -403,7 +445,7 @@ Version: 1.0
     #[tokio::test]
     async fn test_find_package_fallback() {
         // python2.7 is in bullseye but not above
-        let info = find_package("python2.7", "debian", "", None, None)
+        let info = find_package("python2.7", "debian", "", None, None, None)
             .await
             .unwrap();
         assert_eq!(info.stanza.package, "python2.7");
@@ -413,7 +455,7 @@ Version: 1.0
     #[tokio::test]
     async fn test_find_package_devel() {
         // hello is in sid
-        let info = find_package("hello", "debian", "", None, None)
+        let info = find_package("hello", "debian", "", None, None, None)
             .await
             .unwrap();
         assert_eq!(info.stanza.package, "hello");

src/pull.rs

@@ -504,7 +504,7 @@ pub async fn pull(
 
         // Depending on target series, we pick target branch; if latest series is specified,
         // we target the development branch, i.e. the default branch
-        let branch_name = if crate::distro_info::get_ordered_series(package_info.dist.as_str())
+        let branch_name = if crate::distro_info::get_ordered_series_name(package_info.dist.as_str())
             .await?[0]
             != *series
         {
@@ -590,7 +590,7 @@ mod tests {
         let cwd = temp_dir.path();
 
         // Main 'pull' command: the one we want to test
-        let info = crate::package_info::lookup(package, None, series, "", dist, None)
+        let info = crate::package_info::lookup(package, None, series, "", dist, None, None)
             .await
             .unwrap();
         pull(&info, Some(cwd), None, archive.unwrap_or(false))