Compare commits

..

5 Commits

Author SHA1 Message Date
vhaudiquet 62ce4e3696 deb: remove sbuild build mode
CI / build (push) Successful in 8m15s
CI / snap (push) Successful in 3m28s
2026-07-24 14:32:47 +02:00
vhaudiquet a0d35bb18e pull,context: preserve file mtimes when copying sources
CI / build (push) Successful in 8m11s
CI / snap (push) Successful in 6m9s
std::fs::copy resets the destination mtime to "now", which breaks
timestamp-based build systems (autotools/gnulib). Packages like
'hello' from Debian sid ship pre-generated files alongside their
prerequisites; when the copy flattens all mtimes, make considers the
generated targets out-of-date and tries to regenerate them with tools
(e.g. gperf) that are not declared build-dependencies, failing the
build.

Restore the source modification and access times after every file copy
in pull::copy_dir_all, pull::fetch_archive_sources merge step, and
context::unshare copy_dir_recursive/ensure_available.
2026-07-24 14:14:18 +02:00
vhaudiquet e1668d5d80 pull: support Sources.gz without Checksums-Sha256
CI / build (push) Failing after 8m34s
CI / snap (push) Has been skipped
The Ubuntu development series (stonking) Sources.gz no longer ships a
Checksums-Sha256 field, only Checksums-Sha512 and the legacy Files
(MD5) field. The package_info parser only read Checksums-Sha256, so
the file list ended up empty and fetch_orig_tarball panicked on
Option::unwrap() when looking for the orig tarball.

- Add a ChecksumAlgo enum (Md5/Sha256/Sha512) to FileEntry, replacing
  the hardcoded sha256 field, and parse the strongest available
  checksum field (Sha256 > Sha512 > MD5).
- Make download_file_checksum verify against the correct algorithm
  instead of always using SHA-256.
- Replace the unwrap() on the orig tarball search with a proper error
  listing the available files, so future regressions fail clearly
  instead of panicking.
- Add md-5 dependency for MD5 verification.
2026-07-24 12:56:31 +02:00
vhaudiquet f9e11e951b clippy
CI / build (push) Failing after 8m37s
CI / snap (push) Has been skipped
2026-07-24 10:53:27 +02:00
vhaudiquet 768e1c4f78 fmt
CI / build (push) Failing after 1m56s
CI / snap (push) Has been skipped
2026-07-22 14:36:28 +02:00
15 changed files with 224 additions and 144 deletions
+1 -1
View File
@@ -41,7 +41,7 @@ jobs:
- name: Install runtime system dependencies
run: |
sudo apt-get update
sudo apt-get install -y git pristine-tar sbuild mmdebstrap util-linux dpkg-dev
sudo apt-get install -y git pristine-tar mmdebstrap util-linux dpkg-dev
- name: Setup subuid/subgid
run: |
usermod --add-subuids 100000-200000 --add-subgids 100000-200000 ${USER:-root}
+1
View File
@@ -17,6 +17,7 @@ regex = "1"
chrono = "0.4"
tokio = { version = "1.41.1", features = ["full"] }
sha2 = "0.10.8"
md-5 = "0.10"
hex = "0.4.3"
log = "0.4.28"
indicatif = "0.18.3"
+2 -2
View File
@@ -97,7 +97,7 @@ Missing features:
- [x] Build for a specific architecture
- [ ] Three build modes:
- [ ] Build locally (discouraged)
- [x] Build using sbuild+unshare, with binary emulation (default)
- [x] Build using unshare chroot, with binary emulation (default)
- [x] Cross-compilation
- [ ] Async build
- [ ] `pkh status`
@@ -111,7 +111,7 @@ Missing features:
- [ ] Lint the package
- [ ] `pkh test`
- [ ] Run autopkgtest
- [ ] Provide options: local (discouraged), sbuild/VM?, ppa
- [ ] Provide options: local (discouraged), chroot, VM?, ppa
- [ ] Async test
## Nice-to-have features
-1
View File
@@ -31,7 +31,6 @@ parts:
- git
- curl
- pristine-tar
- sbuild
- mmdebstrap
- util-linux
- dpkg-dev
+2 -7
View File
@@ -170,13 +170,8 @@ pub fn parse_changelog_footer(path: &Path) -> Result<(String, String), Box<dyn s
)
})?;
let mut content = String::new();
file.read_to_string(&mut content).map_err(|e| {
format!(
"Failed to read changelog '{}': {}",
path.display(),
e
)
})?;
file.read_to_string(&mut content)
.map_err(|e| format!("Failed to read changelog '{}': {}", path.display(), e))?;
// Find the last maintainer line (format: -- Name <email> Date)
let re = Regex::new(r"--\s*([^<]+?)\s*<([^>]+)>\s*")?;
+1 -4
View File
@@ -168,10 +168,7 @@ mod tests {
ctx.copy_path(&src_dir, &dest_dir).unwrap();
// The regular file was copied.
assert_eq!(
ctx.read_file(&dest_dir.join("real.txt")).unwrap(),
"data"
);
assert_eq!(ctx.read_file(&dest_dir.join("real.txt")).unwrap(), "data");
// The symlink was reproduced as a symlink (not followed).
let meta = std::fs::symlink_metadata(dest_dir.join("dangling")).unwrap();
assert!(meta.file_type().is_symlink());
+53 -11
View File
@@ -48,12 +48,8 @@ fn copy_dir_recursive(src: &Path, dest: &Path) -> io::Result<()> {
Err(e) => {
return Err(io::Error::new(
e.kind(),
format!(
"Failed to read metadata of '{}': {}",
src_path.display(),
e
),
))
format!("Failed to read metadata of '{}': {}", src_path.display(), e),
));
}
};
@@ -81,19 +77,65 @@ fn copy_dir_recursive(src: &Path, dest: &Path) -> io::Result<()> {
// Recursively copy subdirectories
copy_dir_recursive(&src_path, &dest_path)?;
} else {
// Copy regular files
std::fs::copy(&src_path, &dest_path).map_err(|e| {
// Copy regular files, preserving the source modification and
// access times. This is important for autotools/gnulib-based
// packages (e.g. 'hello' from Debian sid) that ship pre-generated
// files alongside their prerequisites: if the copy resets the
// mtime to "now", the prerequisites appear as new as the
// generated targets and `make` tries to regenerate them using
// tools (like gperf) that are not declared build-dependencies.
copy_file_with_times(&src_path, &dest_path)?;
}
}
Ok(())
}
/// Copy a single regular file from `src` to `dest`, preserving the source's
/// modification and access times.
///
/// `std::fs::copy` resets the destination mtime to "now", which breaks
/// timestamp-based build systems (autotools/gnulib) that ship pre-generated
/// files alongside their prerequisites. Restoring the original timestamps
/// prevents `make` from needlessly regenerating those files with tools that
/// may not be installed (e.g. `gperf`).
fn copy_file_with_times(src: &Path, dest: &Path) -> io::Result<()> {
std::fs::copy(src, dest).map_err(|e| {
io::Error::new(
e.kind(),
format!(
"Failed to copy '{}' to '{}': {}",
src_path.display(),
dest_path.display(),
src.display(),
dest.display(),
e
),
)
})?;
// Restore the original timestamps on the destination file.
let metadata = std::fs::metadata(src).map_err(|e| {
io::Error::new(
e.kind(),
format!("Failed to read metadata of '{}': {}", src.display(), e),
)
})?;
let mut times = std::fs::FileTimes::new();
let mut have_times = false;
if let Ok(mtime) = metadata.modified() {
times = times.set_modified(mtime);
have_times = true;
}
if let Ok(atime) = metadata.accessed() {
times = times.set_accessed(atime);
have_times = true;
}
if have_times && let Ok(dest_file) = std::fs::File::open(dest) {
let _ = dest_file.set_times(times).map_err(|e| {
io::Error::new(
e.kind(),
format!("Failed to set times on '{}': {}", dest.display(), e),
)
});
}
Ok(())
@@ -129,7 +171,7 @@ impl ContextDriver for UnshareDriver {
dest_path.display()
);
} else {
std::fs::copy(src, &dest_path)?;
copy_file_with_times(src, &dest_path)?;
debug!("Copied file {} to {}", src.display(), dest_path.display());
}
+1 -1
View File
@@ -336,7 +336,7 @@ impl EphemeralContextGuard {
impl Drop for EphemeralContextGuard {
fn drop(&mut self) {
log::debug!("Cleaning up ephemeral context ({:?})...", &self.chroot_path);
log::debug!("Cleaning up ephemeral context ({:?})...", self.chroot_path);
// Reset to normal context
if let Err(e) = context::manager().set_current(&self.previous_context) {
log::error!("Failed to restore context {}: {}", self.previous_context, e);
+2 -4
View File
@@ -148,12 +148,10 @@ pub async fn build(
)
})?;
if !status.success() {
return Err(
"apt-get update failed inside the build context. \
return Err("apt-get update failed inside the build context. \
If this is a local build, try executing with sudo, \
or re-run with RUST_LOG=debug for more details."
.into(),
);
.into());
}
// Install essential packages
-12
View File
@@ -1,7 +1,6 @@
mod cross;
mod ephemeral;
mod local;
mod sbuild;
use crate::context::{self, Context};
use std::error::Error;
@@ -11,8 +10,6 @@ use std::sync::Arc;
/// Build mode for the binary build
#[derive(PartialEq)]
pub enum BuildMode {
/// Use `sbuild` for the build, configured in unshare mode
Sbuild,
/// Local build, directly on the context
Local,
}
@@ -110,15 +107,6 @@ pub async fn build_binary_package(
)
.await?
}
BuildMode::Sbuild => sbuild::build(
&package,
&version,
arch,
series,
&build_root,
cross,
build_ctx.clone(),
)?,
};
// Retrieve produced .deb files
-44
View File
@@ -1,44 +0,0 @@
/// Sbuild binary package building
/// Call 'sbuild' with the dsc file to build the package with unshare
use crate::context::Context;
use std::error::Error;
use std::path::Path;
use std::sync::Arc;
pub fn build(
package: &str,
version: &str,
arch: &str,
series: &str,
build_root: &str,
cross: bool,
ctx: Arc<Context>,
) -> Result<(), Box<dyn Error>> {
// Find the actual package directory
let package_dir =
crate::deb::find_package_directory(Path::new(build_root), package, version, &ctx)?;
let package_dir_str = package_dir
.to_str()
.ok_or("Invalid package directory path")?;
let mut cmd = ctx.command("sbuild");
cmd.current_dir(package_dir_str);
cmd.arg("--chroot-mode=unshare");
cmd.arg("--no-clean-source");
if cross {
cmd.arg(format!("--host={}", arch));
} else {
cmd.arg(format!("--arch={}", arch));
}
cmd.arg(format!("--dist={}", series));
// Add output directory argument
cmd.arg(format!("--build-dir={}", build_root));
let status = cmd.status()?;
if !status.success() {
return Err(format!("sbuild failed with status: {}", status).into());
}
Ok(())
}
+6 -9
View File
@@ -126,8 +126,7 @@ pub async fn get_ordered_series(dist: &str) -> Result<Vec<SeriesInformation>, Bo
})?;
let series_info = &dist_data.series;
let content = if Path::new(series_info.local.as_str()).exists() {
std::fs::read_to_string(format!("/usr/share/distro-info/{dist}.csv"))
.map_err(|e| {
std::fs::read_to_string(format!("/usr/share/distro-info/{dist}.csv")).map_err(|e| {
format!(
"Failed to read distribution series data for '{dist}' \
from '{}': {}. The 'distro-info' package provides these CSV files.",
@@ -182,7 +181,7 @@ pub async fn get_n_latest_released_series(
}
// Sort by release date descending (newest first)
released_series.sort_by(|a, b| b.release.cmp(&a.release));
released_series.sort_by_key(|b| std::cmp::Reverse(b.release));
Ok(released_series
.iter()
@@ -237,7 +236,8 @@ pub fn get_sources_url(base_url: &str, series: &str, pocket: &str, component: &s
///
/// Example: ubuntu => http://archive.ubuntu.com/ubuntu
pub fn get_base_url(dist: &str) -> Result<String, Box<dyn Error>> {
DATA.dist.get(dist)
DATA.dist
.get(dist)
.map(|d| d.base_url.clone())
.ok_or_else(|| {
format!(
@@ -281,9 +281,7 @@ pub async fn get_keyring_urls(series: &str) -> Result<Vec<String>, Box<dyn Error
Ok(urls)
}
} else {
let series_num = get_debian_series_number(series)
.await?
.ok_or_else(|| {
let series_num = get_debian_series_number(series).await?.ok_or_else(|| {
format!(
"Could not determine the version number for Debian series '{series}'. \
Make sure the 'distro-info' package is installed, which provides the \
@@ -349,8 +347,7 @@ pub async fn get_debian_series_number(series: &str) -> Result<Option<String>, Bo
})?;
let series_info = &dist_data.series;
let content = if Path::new(series_info.local.as_str()).exists() {
std::fs::read_to_string(series_info.local.as_str())
.map_err(|e| {
std::fs::read_to_string(series_info.local.as_str()).map_err(|e| {
format!(
"Failed to read Debian series data from '{}': {}. \
The 'distro-info' package provides this file.",
+4 -6
View File
@@ -71,8 +71,8 @@ fn main() {
.long_help("Inject a package into the build environment before build-dep. Can be a .deb file path, a package name from the archive, or a package from a previously added PPA. Can be specified multiple times.").required(false).action(clap::ArgAction::Append))
.arg(arg!(--cross "Cross-compile for target architecture (instead of qemu-binfmt)")
.long_help("Cross-compile for target architecture (instead of using qemu-binfmt)\nNote that most packages cannot be cross-compiled").required(false))
.arg(arg!(--mode <mode> "Change build mode [sbuild, local]").required(false)
.long_help("Change build mode [sbuild, local]\nDefault will chose depending on other parameters, don't provide if unsure")),
.arg(arg!(--mode <mode> "Change build mode [local]").required(false)
.long_help("Change build mode [local]\nDefault will chose depending on other parameters, don't provide if unsure")),
)
.subcommand(
Command::new("context")
@@ -211,7 +211,6 @@ fn main() {
};
let mode: Option<&str> = sub_matches.get_one::<String>("mode").map(|s| s.as_str());
let mode: Option<pkh::deb::BuildMode> = match mode {
Some("sbuild") => Some(pkh::deb::BuildMode::Sbuild),
Some("local") => Some(pkh::deb::BuildMode::Local),
_ => None,
};
@@ -247,9 +246,8 @@ fn main() {
let context = match type_str {
"local" => ContextConfig::Local,
"ssh" => {
let endpoint = args
.get_one::<String>("endpoint")
.unwrap_or_else(|| {
let endpoint =
args.get_one::<String>("endpoint").unwrap_or_else(|| {
error!(
"An --endpoint is required to create an ssh context. \
Expected format: [ssh://][user@]host[:port]"
+56 -5
View File
@@ -39,8 +39,47 @@ pub struct FileEntry {
pub name: String,
/// Size of the file
pub size: u64,
/// SHA256 hash for the file
pub sha256: String,
/// Checksum hash for the file
pub checksum: String,
/// Algorithm used for the checksum
pub checksum_algo: ChecksumAlgo,
}
/// Checksum algorithm used for a file entry
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ChecksumAlgo {
/// MD5 (legacy 'Files' field)
Md5,
/// SHA-256 ('Checksums-Sha256' field)
Sha256,
/// SHA-512 ('Checksums-Sha512' field)
Sha512,
}
impl ChecksumAlgo {
/// Compute the hex-encoded digest of the given data using this algorithm
pub fn hex_digest(&self, data: &[u8]) -> String {
match self {
ChecksumAlgo::Md5 => {
use md5::{Digest, Md5};
let mut hasher = Md5::new();
hasher.update(data);
hex::encode(hasher.finalize())
}
ChecksumAlgo::Sha256 => {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(data);
hex::encode(hasher.finalize())
}
ChecksumAlgo::Sha512 => {
use sha2::{Digest, Sha512};
let mut hasher = Sha512::new();
hasher.update(data);
hex::encode(hasher.finalize())
}
}
}
}
/// A package 'stanza' as found is 'Sources.gz' files, containing basic information about a source package
@@ -138,16 +177,28 @@ impl Iterator for DebianSources {
return self.next();
}
// Parse package files
// Parse package files.
// Prefer the strongest available checksum field: Checksums-Sha256,
// then Checksums-Sha512, then the legacy 'Files' (MD5) field.
// Some archives (e.g. the Ubuntu development series) no longer ship
// Checksums-Sha256, so falling back is required to keep working.
let mut files = Vec::new();
if let Some(checksums) = fields.get("Checksums-Sha256") {
let (checksum_field, algo) = if fields.contains_key("Checksums-Sha256") {
("Checksums-Sha256", ChecksumAlgo::Sha256)
} else if fields.contains_key("Checksums-Sha512") {
("Checksums-Sha512", ChecksumAlgo::Sha512)
} else {
("Files", ChecksumAlgo::Md5)
};
if let Some(checksums) = fields.get(checksum_field) {
for line in checksums.lines() {
let parts: Vec<&str> = line.split_whitespace().collect();
if parts.len() >= 3 {
files.push(FileEntry {
sha256: parts[0].to_string(),
checksum: parts[0].to_string(),
size: parts[1].parse().unwrap_or(0),
name: parts[2].to_string(),
checksum_algo: algo,
});
}
}
+73 -15
View File
@@ -83,7 +83,6 @@ fn clone_repo(
}
}
use sha2::{Digest, Sha256};
use std::fs::File;
use std::io::Write;
@@ -104,20 +103,56 @@ fn copy_dir_all(src: &Path, dst: &Path) -> Result<(), Box<dyn Error>> {
// Reproduce symlinks as symlinks rather than following them, so that
// dangling/absolute symlinks do not abort the copy.
if std::fs::symlink_metadata(&src_path)?.file_type().is_symlink() {
if std::fs::symlink_metadata(&src_path)?
.file_type()
.is_symlink()
{
let target = std::fs::read_link(&src_path)?;
let _ = std::fs::remove_file(&dst_path);
symlink(&target, &dst_path)?;
} else if src_path.is_dir() {
copy_dir_all(&src_path, &dst_path)?;
} else {
// Copy the file, preserving the source modification and access
// times. This is important for autotools/gnulib-based packages
// (e.g. 'hello' from Debian sid) that ship pre-generated files
// alongside their prerequisites: if the copy resets the mtime to
// "now", the prerequisites appear as new as the generated targets
// and `make` tries to regenerate them using tools (like gperf)
// that are not declared build-dependencies.
std::fs::copy(&src_path, &dst_path)?;
copy_file_times(&src_path, &dst_path)?;
}
}
Ok(())
}
/// Restore the modification and access times of `dest` to match `src`.
///
/// `std::fs::copy` resets the destination mtime to "now", which breaks
/// timestamp-based build systems (autotools/gnulib) that ship pre-generated
/// files alongside their prerequisites. Restoring the original timestamps
/// prevents `make` from needlessly regenerating those files with tools that
/// may not be installed (e.g. `gperf`).
fn copy_file_times(src: &Path, dest: &Path) -> Result<(), Box<dyn Error>> {
let metadata = std::fs::metadata(src)?;
let mut times = std::fs::FileTimes::new();
let mut have_times = false;
if let Ok(mtime) = metadata.modified() {
times = times.set_modified(mtime);
have_times = true;
}
if let Ok(atime) = metadata.accessed() {
times = times.set_accessed(atime);
have_times = true;
}
if have_times && let Ok(dest_file) = std::fs::File::open(dest) {
let _ = dest_file.set_times(times);
}
Ok(())
}
/// Helper function to extract tar archive with progress tracking
fn extract_tar_archive<D, F>(
file_path: &Path,
@@ -218,18 +253,19 @@ fn checkout_pristine_tar(package_dir: &Path, filename: &str) -> Result<(), Box<d
async fn download_file_checksum(
url: &str,
checksum: &str,
algo: crate::package_info::ChecksumAlgo,
target_dir: &Path,
progress: ProgressCallback<'_>,
) -> Result<(), Box<dyn Error>> {
// Download with reqwest
let response = reqwest::get(url).await?;
if !response.status().is_success() {
return Err(format!("Failed to download '{}' : {}", &url, response.status()).into());
return Err(format!("Failed to download '{}' : {}", url, response.status()).into());
}
let total_size = response
.content_length()
.ok_or(format!("Failed to get content length from '{}'", &url))?;
.ok_or(format!("Failed to get content length from '{}'", url))?;
let mut index = 0;
// Target file: extract file name from URL
@@ -239,11 +275,13 @@ async fn download_file_checksum(
// Download chunk by chunk to disk, while updating hasher for checksum
let mut stream = response.bytes_stream();
let mut hasher = Sha256::new();
// Accumulate the downloaded bytes so we can compute the final digest with the
// correct algorithm once the download is complete.
let mut buffer: Vec<u8> = Vec::with_capacity(total_size as usize);
while let Some(item) = stream.next().await {
let chunk = item?;
file.write_all(&chunk)?;
hasher.update(&chunk);
buffer.extend_from_slice(&chunk);
if let Some(cb) = progress {
index = min(index + chunk.len(), total_size as usize);
@@ -251,9 +289,8 @@ async fn download_file_checksum(
}
}
// Verify checksum
let result = hasher.finalize();
let calculated_checksum = hex::encode(result);
// Verify checksum using the algorithm specified for this file
let calculated_checksum = algo.hex_digest(&buffer);
if calculated_checksum != checksum {
return Err(format!(
"Checksum mismatch! Expected {}, got {}",
@@ -320,7 +357,18 @@ async fn fetch_orig_tarball(
.files
.iter()
.find(|f| f.name.contains(".orig.tar."))
.unwrap();
.ok_or_else(|| {
format!(
"Could not find orig tarball in file list for package '{}'. \
Available files: {:?}",
info.stanza.package,
info.stanza
.files
.iter()
.map(|f| &f.name)
.collect::<Vec<_>>()
)
})?;
let filename = &orig_file.name;
// 1. Try executing pristine-tar
@@ -339,8 +387,9 @@ async fn fetch_orig_tarball(
// or the current directory if cwd is None (which effectively is the parent of the package dir)
let target_dir = cwd.unwrap_or_else(|| Path::new("."));
download_file_checksum(
format!("{}/{}", &info.archive_url, filename).as_str(),
&orig_file.sha256,
format!("{}/{}", info.archive_url, filename).as_str(),
&orig_file.checksum,
orig_file.checksum_algo,
target_dir,
progress,
)
@@ -369,8 +418,9 @@ async fn fetch_dsc_file(
debug!("Fetching dsc file: {}", filename);
download_file_checksum(
format!("{}/{}", &info.archive_url, filename).as_str(),
&dsc_file.sha256,
format!("{}/{}", info.archive_url, filename).as_str(),
&dsc_file.checksum,
dsc_file.checksum_algo,
target_dir,
progress,
)
@@ -394,7 +444,14 @@ async fn fetch_archive_sources(
for file in &info.stanza.files {
let url = format!("{}/{}", info.archive_url, file.name);
download_file_checksum(&url, &file.sha256, package_dir, progress).await?;
download_file_checksum(
&url,
&file.checksum,
file.checksum_algo,
package_dir,
progress,
)
.await?;
// Extract all tar archives, merging extracted directories
if file.name.ends_with(".tar.gz") || file.name.ends_with(".tar.xz") {
@@ -443,6 +500,7 @@ async fn fetch_archive_sources(
copy_dir_all(&sub_path, &target_path)?;
} else {
std::fs::copy(&sub_path, &target_path)?;
copy_file_times(&sub_path, &target_path)?;
}
}
std::fs::remove_dir_all(&src_dir)?;