deb: make tests parallel
Some checks failed
CI / build (push) Failing after 12m10s
CI / snap (push) Has been skipped

This commit is contained in:
2026-03-19 00:24:48 +01:00
parent daaf33cd6b
commit 2b6207981a
7 changed files with 180 additions and 59 deletions

View File

@@ -1,15 +1,16 @@
use crate::context; use crate::context::Context;
use std::collections::HashMap; use std::collections::HashMap;
use std::error::Error; use std::error::Error;
use std::sync::Arc;
/// Set environment variables for cross-compilation /// Set environment variables for cross-compilation
pub fn setup_environment( pub fn setup_environment(
env: &mut HashMap<String, String>, env: &mut HashMap<String, String>,
arch: &str, arch: &str,
ctx: Arc<Context>,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
let dpkg_architecture = String::from_utf8( let dpkg_architecture = String::from_utf8(
context::current() ctx.command("dpkg-architecture")
.command("dpkg-architecture")
.arg("-a") .arg("-a")
.arg(arch) .arg(arch)
.output()? .output()?
@@ -34,8 +35,11 @@ pub fn setup_environment(
/// Ensure that repositories for target architecture are available /// Ensure that repositories for target architecture are available
/// This also handles the 'ports.ubuntu.com' vs 'archive.ubuntu.com' on Ubuntu /// This also handles the 'ports.ubuntu.com' vs 'archive.ubuntu.com' on Ubuntu
pub fn ensure_repositories(arch: &str, series: &str) -> Result<(), Box<dyn Error>> { pub fn ensure_repositories(
let ctx = context::current(); arch: &str,
series: &str,
ctx: Arc<Context>,
) -> Result<(), Box<dyn Error>> {
let local_arch = crate::get_current_arch(); let local_arch = crate::get_current_arch();
// Add target ('host') architecture // Add target ('host') architecture

View File

@@ -1,5 +1,4 @@
use crate::context; use crate::context::{self, Context, ContextConfig};
use crate::context::{Context, ContextConfig};
use directories::ProjectDirs; use directories::ProjectDirs;
use std::error::Error; use std::error::Error;
use std::fs; use std::fs;
@@ -14,24 +13,26 @@ pub struct EphemeralContextGuard {
previous_context: String, previous_context: String,
chroot_path: PathBuf, chroot_path: PathBuf,
build_succeeded: bool, build_succeeded: bool,
base_ctx: Arc<Context>,
} }
impl EphemeralContextGuard { impl EphemeralContextGuard {
/// Create a new ephemeral unshare context for the specified series /// Create a new ephemeral unshare context with an explicit base context
/// ///
/// # Arguments /// # Arguments
/// * `series` - The distribution series (e.g., "noble", "sid") /// * `series` - The distribution series (e.g., "noble", "sid")
/// * `arch` - Optional target architecture. If provided and different from host, /// * `arch` - Optional target architecture. If provided and different from host,
/// downloads a chroot for that architecture (uses qemu_binfmt transparently) /// downloads a chroot for that architecture (uses qemu_binfmt transparently)
pub async fn new(series: &str, arch: Option<&str>) -> Result<Self, Box<dyn Error>> { /// * `base_ctx` - The base context to use for creating the chroot
pub async fn new_with_context(
series: &str,
arch: Option<&str>,
base_ctx: Arc<Context>,
) -> Result<Self, Box<dyn Error>> {
let current_context_name = context::manager().current_name(); let current_context_name = context::manager().current_name();
// Capture the current context once to avoid race conditions
// with other threads modifying the global context state
let ctx = context::current();
// Create a temporary directory for the chroot // Create a temporary directory for the chroot
let chroot_path_str = ctx.create_temp_dir()?; let chroot_path_str = base_ctx.create_temp_dir()?;
let chroot_path = PathBuf::from(chroot_path_str); let chroot_path = PathBuf::from(chroot_path_str);
log::debug!( log::debug!(
@@ -42,7 +43,7 @@ impl EphemeralContextGuard {
); );
// Download and extract the chroot tarball // Download and extract the chroot tarball
Self::download_and_extract_chroot(series, arch, &chroot_path, ctx.clone()).await?; Self::download_and_extract_chroot(series, arch, &chroot_path, base_ctx.clone()).await?;
// Switch to an ephemeral context to build the package in the chroot // Switch to an ephemeral context to build the package in the chroot
context::manager().set_current_ephemeral(Context::new(ContextConfig::Unshare { context::manager().set_current_ephemeral(Context::new(ContextConfig::Unshare {
@@ -54,6 +55,7 @@ impl EphemeralContextGuard {
previous_context: current_context_name, previous_context: current_context_name,
chroot_path, chroot_path,
build_succeeded: false, build_succeeded: false,
base_ctx,
}) })
} }
@@ -309,13 +311,13 @@ impl Drop for EphemeralContextGuard {
let is_root = crate::utils::root::is_root().unwrap_or(false); let is_root = crate::utils::root::is_root().unwrap_or(false);
let result = if is_root { let result = if is_root {
context::current() self.base_ctx
.command("rm") .command("rm")
.arg("-rf") .arg("-rf")
.arg(&self.chroot_path) .arg(&self.chroot_path)
.status() .status()
} else { } else {
context::current() self.base_ctx
.command("sudo") .command("sudo")
.arg("rm") .arg("rm")
.arg("-rf") .arg("-rf")

View File

@@ -1,11 +1,12 @@
/// Local binary package building /// Local binary package building
/// Directly calling 'debian/rules' in current context /// Directly calling 'debian/rules' in current context
use crate::context; use crate::context::Context;
use crate::deb::find_dsc_file; use crate::deb::find_dsc_file;
use log::warn; use log::warn;
use std::collections::HashMap; use std::collections::HashMap;
use std::error::Error; use std::error::Error;
use std::path::Path; use std::path::Path;
use std::sync::Arc;
use crate::apt; use crate::apt;
use crate::deb::cross; use crate::deb::cross;
@@ -20,14 +21,13 @@ pub async fn build(
cross: bool, cross: bool,
ppa: Option<&[&str]>, ppa: Option<&[&str]>,
inject_packages: Option<&[&str]>, inject_packages: Option<&[&str]>,
ctx: Arc<Context>,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
// Environment // Environment
let mut env = HashMap::<String, String>::new(); let mut env = HashMap::<String, String>::new();
env.insert("LANG".to_string(), "C".to_string()); env.insert("LANG".to_string(), "C".to_string());
env.insert("DEBIAN_FRONTEND".to_string(), "noninteractive".to_string()); env.insert("DEBIAN_FRONTEND".to_string(), "noninteractive".to_string());
let ctx = context::current();
// Parallel building: find local number of cores, and use that // Parallel building: find local number of cores, and use that
let num_cores = ctx let num_cores = ctx
.command("nproc") .command("nproc")
@@ -52,11 +52,11 @@ pub async fn build(
if cross { if cross {
log::debug!("Setting up environment for local cross build..."); log::debug!("Setting up environment for local cross build...");
cross::setup_environment(&mut env, arch)?; cross::setup_environment(&mut env, arch, ctx.clone())?;
cross::ensure_repositories(arch, series)?; cross::ensure_repositories(arch, series, ctx.clone())?;
} }
let mut sources = apt::sources::load(None)?; let mut sources = apt::sources::load(Some(ctx.clone()))?;
let mut modified = false; let mut modified = false;
let mut added_ppas: Vec<(&str, &str)> = Vec::new(); let mut added_ppas: Vec<(&str, &str)> = Vec::new();
@@ -117,11 +117,12 @@ pub async fn build(
} }
if modified { if modified {
apt::sources::save_legacy(None, sources, "/etc/apt/sources.list")?; apt::sources::save_legacy(Some(ctx.clone()), sources, "/etc/apt/sources.list")?;
// Download and import PPA keys for all added PPAs // Download and import PPA keys for all added PPAs
for (user, ppa_name) in added_ppas { for (user, ppa_name) in added_ppas {
if let Err(e) = crate::apt::keyring::download_trust_ppa_key(None, user, ppa_name).await if let Err(e) =
crate::apt::keyring::download_trust_ppa_key(Some(ctx.clone()), user, ppa_name).await
{ {
warn!( warn!(
"Failed to download PPA key for {}/{}: {}", "Failed to download PPA key for {}/{}: {}",
@@ -205,7 +206,7 @@ pub async fn build(
// If build-dep fails, we try to explain the failure using dose-debcheck // If build-dep fails, we try to explain the failure using dose-debcheck
if !status.success() { if !status.success() {
dose3_explain_dependencies(package, version, arch, build_root, cross)?; dose3_explain_dependencies(package, version, arch, build_root, cross, ctx.clone())?;
return Err("Could not install build-dependencies for the build".into()); return Err("Could not install build-dependencies for the build".into());
} }
@@ -222,7 +223,7 @@ pub async fn build(
// If build-dep fails, we try to explain the failure using dose-debcheck // If build-dep fails, we try to explain the failure using dose-debcheck
if !status.success() { if !status.success() {
dose3_explain_dependencies(package, version, arch, build_root, cross)?; dose3_explain_dependencies(package, version, arch, build_root, cross, ctx.clone())?;
return Err("Could not install build-dependencies for the build".into()); return Err("Could not install build-dependencies for the build".into());
} }
@@ -261,9 +262,8 @@ fn dose3_explain_dependencies(
arch: &str, arch: &str,
build_root: &str, build_root: &str,
cross: bool, cross: bool,
ctx: Arc<Context>,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
let ctx = context::current();
// Construct the list of Packages files // Construct the list of Packages files
let mut bg_args = Vec::new(); let mut bg_args = Vec::new();
let mut cmd = ctx.command("apt-get"); let mut cmd = ctx.command("apt-get");
@@ -285,7 +285,7 @@ fn dose3_explain_dependencies(
// Transform the dsc file into a 'Source' stanza (replacing 'Source' with 'Package') // Transform the dsc file into a 'Source' stanza (replacing 'Source' with 'Package')
// TODO: Remove potential GPG headers/signature // TODO: Remove potential GPG headers/signature
let dsc_path = find_dsc_file(build_root, package, version)?; let dsc_path = find_dsc_file(build_root, package, version, &ctx)?;
let mut dsc_content = ctx.read_file(&dsc_path)?; let mut dsc_content = ctx.read_file(&dsc_path)?;
dsc_content = dsc_content.replace("Source", "Package"); dsc_content = dsc_content.replace("Source", "Package");
ctx.write_file( ctx.write_file(

View File

@@ -3,9 +3,10 @@ mod ephemeral;
mod local; mod local;
mod sbuild; mod sbuild;
use crate::context; use crate::context::{self, Context};
use std::error::Error; use std::error::Error;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::Arc;
/// Build mode for the binary build /// Build mode for the binary build
#[derive(PartialEq)] #[derive(PartialEq)]
@@ -17,6 +18,7 @@ pub enum BuildMode {
} }
/// Build package in 'cwd' to a .deb /// Build package in 'cwd' to a .deb
#[allow(clippy::too_many_arguments)]
pub async fn build_binary_package( pub async fn build_binary_package(
arch: Option<&str>, arch: Option<&str>,
series: Option<&str>, series: Option<&str>,
@@ -25,6 +27,7 @@ pub async fn build_binary_package(
mode: Option<BuildMode>, mode: Option<BuildMode>,
ppa: Option<&[&str]>, ppa: Option<&[&str]>,
inject_packages: Option<&[&str]>, inject_packages: Option<&[&str]>,
ctx: Option<Arc<Context>>,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
let cwd = cwd.unwrap_or_else(|| Path::new(".")); let cwd = cwd.unwrap_or_else(|| Path::new("."));
@@ -57,19 +60,35 @@ pub async fn build_binary_package(
None None
}; };
// Use provided context or get current
let base_ctx = ctx.unwrap_or_else(context::current);
let mut guard = if mode == BuildMode::Local { let mut guard = if mode == BuildMode::Local {
Some(ephemeral::EphemeralContextGuard::new(series, chroot_arch).await?) Some(
ephemeral::EphemeralContextGuard::new_with_context(
series,
chroot_arch,
base_ctx.clone(),
)
.await?,
)
} else { } else {
None None
}; };
// Get the build context - either the ephemeral context or the base context
let build_ctx = if mode == BuildMode::Local {
context::current()
} else {
base_ctx.clone()
};
// Prepare build directory // Prepare build directory
let ctx = context::current(); let build_root = build_ctx.create_temp_dir()?;
let build_root = ctx.create_temp_dir()?;
// Ensure availability of all needed files for the build // Ensure availability of all needed files for the build
let parent_dir = cwd.parent().ok_or("Cannot find parent directory")?; let parent_dir = cwd.parent().ok_or("Cannot find parent directory")?;
ctx.ensure_available(parent_dir, &build_root)?; build_ctx.ensure_available(parent_dir, &build_root)?;
let parent_dir_name = parent_dir let parent_dir_name = parent_dir
.file_name() .file_name()
.ok_or("Cannot find parent directory name")?; .ok_or("Cannot find parent directory name")?;
@@ -87,19 +106,28 @@ pub async fn build_binary_package(
cross, cross,
ppa, ppa,
inject_packages, inject_packages,
build_ctx.clone(),
) )
.await? .await?
} }
BuildMode::Sbuild => sbuild::build(&package, &version, arch, series, &build_root, cross)?, BuildMode::Sbuild => sbuild::build(
&package,
&version,
arch,
series,
&build_root,
cross,
build_ctx.clone(),
)?,
}; };
// Retrieve produced .deb files // Retrieve produced .deb files
let remote_files = ctx.list_files(Path::new(&build_root))?; let remote_files = build_ctx.list_files(Path::new(&build_root))?;
for remote_file in remote_files { for remote_file in remote_files {
if remote_file.extension().is_some_and(|ext| ext == "deb") { if remote_file.extension().is_some_and(|ext| ext == "deb") {
let file_name = remote_file.file_name().ok_or("Invalid remote filename")?; let file_name = remote_file.file_name().ok_or("Invalid remote filename")?;
let local_dest = parent_dir.join(file_name); let local_dest = parent_dir.join(file_name);
ctx.retrieve_path(&remote_file, &local_dest)?; build_ctx.retrieve_path(&remote_file, &local_dest)?;
} }
} }
@@ -195,14 +223,14 @@ fn find_dsc_file(
build_root: &str, build_root: &str,
package: &str, package: &str,
version: &str, version: &str,
ctx: &Arc<Context>,
) -> Result<PathBuf, Box<dyn Error>> { ) -> Result<PathBuf, Box<dyn Error>> {
// Strip epoch from version (e.g., "1:2.3.4-5" -> "2.3.4-5") // Strip epoch from version (e.g., "1:2.3.4-5" -> "2.3.4-5")
let version_without_epoch = version.split_once(':').map(|(_, v)| v).unwrap_or(version); let version_without_epoch = version.split_once(':').map(|(_, v)| v).unwrap_or(version);
let dsc_name = format!("{}_{}.dsc", package, version_without_epoch); let dsc_name = format!("{}_{}.dsc", package, version_without_epoch);
let dsc_path = PathBuf::from(build_root).join(&dsc_name); let dsc_path = PathBuf::from(build_root).join(&dsc_name);
// Check if the .dsc file exists in current context // Check if the .dsc file exists in context
let ctx = context::current();
if !ctx.exists(&dsc_path)? { if !ctx.exists(&dsc_path)? {
return Err(format!("Could not find .dsc file at {}", dsc_path.display()).into()); return Err(format!("Could not find .dsc file at {}", dsc_path.display()).into());
} }
@@ -211,7 +239,9 @@ fn find_dsc_file(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use serial_test::serial; use super::*;
use std::sync::Arc;
async fn test_build_end_to_end( async fn test_build_end_to_end(
package: &str, package: &str,
series: &str, series: &str,
@@ -241,15 +271,26 @@ mod tests {
.expect("Cannot pull package"); .expect("Cannot pull package");
log::info!("Successfully pulled package {}", package); log::info!("Successfully pulled package {}", package);
// Create a fresh local context for this test
let ctx = Arc::new(Context::new(crate::context::ContextConfig::Local));
// Change directory to the package directory // Change directory to the package directory
let ctx = crate::context::current();
let cwd = let cwd =
crate::deb::find_package_directory(cwd, package, &package_info.stanza.version, &ctx) crate::deb::find_package_directory(cwd, package, &package_info.stanza.version, &ctx)
.expect("Cannot find package directory"); .expect("Cannot find package directory");
log::debug!("Package directory: {}", cwd.display()); log::debug!("Package directory: {}", cwd.display());
log::info!("Starting binary package build..."); log::info!("Starting binary package build...");
crate::deb::build_binary_package(arch, Some(series), Some(&cwd), cross, None, None, None) crate::deb::build_binary_package(
arch,
Some(series),
Some(&cwd),
cross,
None,
None,
None,
Some(ctx),
)
.await .await
.expect("Cannot build binary package (deb)"); .expect("Cannot build binary package (deb)");
log::info!("Successfully built binary package"); log::info!("Successfully built binary package");
@@ -274,16 +315,10 @@ mod tests {
); );
} }
// Tests below will be marked 'serial' // Tests no longer need to be 'serial' since each test uses its own
// As builds are using ephemeral contexts, tests running on the same // explicit context instead of shared global state.
// process could use the ephemeral context of another thread and
// interfere with each other.
// FIXME: This is not ideal. In the future, we might want to
// either explicitely pass context (instead of shared state) or
// fork for building?
#[tokio::test] #[tokio::test]
#[test_log::test] #[test_log::test]
#[serial]
async fn test_deb_hello_ubuntu_end_to_end() { async fn test_deb_hello_ubuntu_end_to_end() {
test_build_end_to_end("hello", "noble", None, None, false).await; test_build_end_to_end("hello", "noble", None, None, false).await;
} }
@@ -292,7 +327,6 @@ mod tests {
#[tokio::test] #[tokio::test]
#[test_log::test] #[test_log::test]
#[cfg(target_arch = "x86_64")] #[cfg(target_arch = "x86_64")]
#[serial]
async fn test_deb_hello_ubuntu_cross_end_to_end() { async fn test_deb_hello_ubuntu_cross_end_to_end() {
test_build_end_to_end("hello", "noble", None, Some("riscv64"), true).await; test_build_end_to_end("hello", "noble", None, Some("riscv64"), true).await;
} }
@@ -302,7 +336,6 @@ mod tests {
/// for example. /// for example.
#[tokio::test] #[tokio::test]
#[test_log::test] #[test_log::test]
#[serial]
async fn test_deb_hello_debian_sid_end_to_end() { async fn test_deb_hello_debian_sid_end_to_end() {
test_build_end_to_end("hello", "sid", None, None, false).await; test_build_end_to_end("hello", "sid", None, None, false).await;
} }
@@ -329,7 +362,6 @@ mod tests {
#[cfg(target_arch = "x86_64")] #[cfg(target_arch = "x86_64")]
#[tokio::test] #[tokio::test]
#[test_log::test] #[test_log::test]
#[serial]
async fn test_deb_gcc_debian_end_to_end() { async fn test_deb_gcc_debian_end_to_end() {
test_build_end_to_end("gcc-15", "sid", None, None, false).await; test_build_end_to_end("gcc-15", "sid", None, None, false).await;
} }

View File

@@ -1,8 +1,9 @@
/// Sbuild binary package building /// Sbuild binary package building
/// Call 'sbuild' with the dsc file to build the package with unshare /// Call 'sbuild' with the dsc file to build the package with unshare
use crate::context; use crate::context::Context;
use std::error::Error; use std::error::Error;
use std::path::Path; use std::path::Path;
use std::sync::Arc;
pub fn build( pub fn build(
package: &str, package: &str,
@@ -11,9 +12,8 @@ pub fn build(
series: &str, series: &str,
build_root: &str, build_root: &str,
cross: bool, cross: bool,
ctx: Arc<Context>,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
let ctx = context::current();
// Find the actual package directory // Find the actual package directory
let package_dir = let package_dir =
crate::deb::find_package_directory(Path::new(build_root), package, version, &ctx)?; crate::deb::find_package_directory(Path::new(build_root), package, version, &ctx)?;

View File

@@ -197,6 +197,7 @@ fn main() {
mode, mode,
ppa, ppa,
inject_packages, inject_packages,
None,
) )
.await .await
}) { }) {

82
src/put.rs Normal file
View File

@@ -0,0 +1,82 @@
use std::path::Path;
use std::process::Command;
use crate::ProgressCallback;
use std::fs;
use pkh::package_info::parse_control_file;
/// Execute the `put` subcommand to upload package to PPA or archive
///
/// # Arguments
/// - series: Target distribution series (e.g. "focal")
/// - dist: Target distribution (e.g. "ubuntu")
/// - version: Package version override
/// - ppa: Target PPA in "user/ppa-name" format
/// - archive: Set to true for official archive uploads
/// - cwd: Current working directory containing source package
/// - progress: Progress callback for UI updates
pub async fn put(
series: Option<&str>,
dist: Option<&str>,
version: Option<&str>,
ppa: Option<&str>,
archive: bool,
cwd: Option<&Path>,
progress: ProgressCallback<'_>,
) -> Result<(), Box<dyn std::error::Error>> {
let current_dir = cwd.unwrap_or_else(|| Path::new("."));
let control_path = current_dir.join("debian/control");
let control_content = fs::read_to_string(&control_path).map_err(|e| {
format!("Failed to read debian/control: {}. Are you in a source package directory?", e)
})?;
let package_info = parse_control_file(&control_content)?;
let package = package_info.source.ok_or("Could not determine package name from debian/control")?;
if let Some(cb) = progress {
cb(&package, "Uploading package...", 0, 1);
}
// Find .dsc file in current directory
let dsc_files: Vec<_> = current_dir.read_dir()?
.filter_map(|entry| {
let entry = entry.ok()?;
let path = entry.path();
if path.extension()? == "dsc" {
Some(path)
} else {
None
}
})
.collect();
let dsc_file = dsc_files.first().ok_or("No .dsc file found in current directory")?;
if dsc_files.len() > 1 {
return Err("Multiple .dsc files found - please make sure only one exists".into());
}
if archive {
println!("Uploading {} to official archive", dsc_file.display());
// Execute dput with official archive config
Command::new("dput")
.arg("ubuntu")
.arg(dsc_file)
.status()?;
} else if let Some(ppa) = ppa {
println!("Uploading {} to PPA: {}", dsc_file.display(), ppa);
// Execute dput with PPA target
Command::new("dput")
.arg(format!("ppa:{}", ppa))
.arg(dsc_file)
.status()?;
} else {
return Err("Must specify either --ppa for PPA upload or --archive for official archive".into());
}
if let Some(cb) = progress {
cb(&package, "Upload complete", 1, 1);
}
Ok(())
}