Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c1f8893576 | ||
|
|
e640b153bd | ||
|
|
231c478d0b | ||
|
|
1aa0ca3d2f | ||
|
|
b20acf3199 | ||
|
|
3feca504fc | ||
|
|
85f0d7d92f | ||
|
|
c45edcee76 | ||
|
|
1fc1d1aa77 | ||
|
|
af870cb7cb |
+109
-196
@@ -16,8 +16,6 @@ use std::sync::Arc;
|
||||
use crate::context::Context;
|
||||
use crate::debian::{ChecksumEntry, ControlInfo, FileChecksums, FilesList};
|
||||
|
||||
use super::parse_checksum_field;
|
||||
|
||||
/// Digests of one artifact.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
struct ArtifactHashes {
|
||||
@@ -57,7 +55,7 @@ pub struct BinaryMetadataOptions {
|
||||
/// `dpkg-genchanges -b`: sorted `Binary` list, encounter-order `Architecture`
|
||||
/// accumulation, sorted `Description` lines formatted like dpkg, `.buildinfo`
|
||||
/// registration in `debian/files`, and binary-NMU handling (`Source:
|
||||
/// pkg (prev)` + previous `.dsc` redistribution when present).
|
||||
/// pkg (prev)` + `Binary-Only-Changes`, with no source files distributed).
|
||||
pub fn generate_binary_metadata(
|
||||
ctx: &Arc<Context>,
|
||||
package_dir: &Path,
|
||||
@@ -137,28 +135,24 @@ pub fn generate_binary_metadata(
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Binary-NMU: redistribute the previous source when present
|
||||
// Binary-NMU: reference the previous source version, textually only
|
||||
// ------------------------------------------------------------------
|
||||
let sversion = entry.version.no_epoch();
|
||||
let mut source_display = entry.source.clone();
|
||||
let mut binary_only_changes = None;
|
||||
|
||||
if entry.binary_only {
|
||||
// A binary-only upload must reference the previous source version;
|
||||
// a changelog that cannot yield it is a hard error, like in the
|
||||
// source-build path. Reuse the changelog read above instead of
|
||||
// reading the file a second time.
|
||||
// Like dpkg-genchanges/genbuildinfo, a binary-only upload references
|
||||
// the previous source version in the `Source` field and records the
|
||||
// entry in `Binary-Only-Changes`, but distributes NO source files:
|
||||
// the previous `.dsc` and its tarballs already sit in the archive,
|
||||
// and are not re-uploaded even when present next to the tree.
|
||||
if let Some(prev) = &previous_entry {
|
||||
source_display = format!("{} ({})", entry.source, prev.version.full());
|
||||
binary_only_changes = Some(format!(
|
||||
"{}\n\n -- {} <{}> {}",
|
||||
entry.changes_field, entry.maintainer_name, entry.maintainer_email, entry.date_raw
|
||||
));
|
||||
let dsc_name = format!("{}_{}.dsc", entry.source, prev.version.no_epoch());
|
||||
let dsc_path = upload_dir.join(&dsc_name);
|
||||
if ctx.exists(&dsc_path)? {
|
||||
include_dsc_artifacts(ctx, upload_dir, &dsc_name, &mut checksums)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,6 +291,7 @@ pub fn generate_binary_metadata(
|
||||
date: entry.date_raw.clone(),
|
||||
source: source_display,
|
||||
binaries,
|
||||
binary_only: entry.binary_only,
|
||||
built_for_profiles: opts.profiles.clone(),
|
||||
architecture: arch_values.join(" "),
|
||||
version: entry.version.full(),
|
||||
@@ -400,90 +395,6 @@ fn hashes_in_context(
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Pull the `.dsc` checksums (and its referenced tarballs) into the
|
||||
/// checksum registry, mirroring how binary-NMU uploads redistribute the
|
||||
/// previous source.
|
||||
fn include_dsc_artifacts(
|
||||
ctx: &Arc<Context>,
|
||||
upload_dir: &Path,
|
||||
dsc_name: &str,
|
||||
checksums: &mut FileChecksums,
|
||||
) -> Result<(), Box<dyn Error>> {
|
||||
let dsc_content = ctx.read_file(&upload_dir.join(dsc_name))?;
|
||||
let para = crate::debian::control::parse_paragraphs(
|
||||
crate::debian::control::strip_clearsigned_armour(&dsc_content),
|
||||
)
|
||||
.into_iter()
|
||||
.next()
|
||||
.ok_or_else(|| format!("'{dsc_name}' is empty"))?;
|
||||
|
||||
// Names and partial checksums are filled from the very same validated
|
||||
// lines, so a listed name can never miss its checksum entry.
|
||||
// Distribution order follows the Checksums fields (Checksums-Sha1 then
|
||||
// Checksums-Sha256), like dpkg-genchanges; the `Files` field only
|
||||
// supplements the md5 digests.
|
||||
let mut names: Vec<String> = Vec::new();
|
||||
let mut partials: BTreeMap<String, super::PartialChecksum> = BTreeMap::new();
|
||||
for field in ["Checksums-Sha1", "Checksums-Sha256", "Files"] {
|
||||
let Some(value) = para.get(field) else {
|
||||
continue;
|
||||
};
|
||||
for cl in parse_checksum_field(field, value)
|
||||
.map_err(|e| format!("cannot parse '{dsc_name}': {e}"))?
|
||||
{
|
||||
let slot = partials.entry(cl.name.clone()).or_default();
|
||||
match field {
|
||||
"Checksums-Sha1" => slot.sha1 = Some(cl.digest),
|
||||
"Checksums-Sha256" => slot.sha256 = Some(cl.digest),
|
||||
_ => slot.md5 = Some(cl.digest),
|
||||
}
|
||||
slot.size = Some(cl.size);
|
||||
if field != "Files" && !names.contains(&cl.name) {
|
||||
names.push(cl.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The .dsc itself is hashed fresh (it may be signed/rewritten); the
|
||||
// tarballs reuse the .dsc-recorded digests, like dpkg-genchanges does.
|
||||
let dsc_hashes =
|
||||
hashes_in_context(ctx, upload_dir, std::slice::from_ref(&dsc_name.to_string()))?;
|
||||
if let Some(h) = dsc_hashes.get(dsc_name) {
|
||||
checksums.insert_entry(
|
||||
dsc_name,
|
||||
ChecksumEntry {
|
||||
size: h.size,
|
||||
md5: h.md5.clone(),
|
||||
sha1: h.sha1.clone(),
|
||||
sha256: h.sha256.clone(),
|
||||
// No SHA-512 digest available (see above).
|
||||
sha512: String::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
for name in &names {
|
||||
if name == dsc_name {
|
||||
continue;
|
||||
}
|
||||
let p = partials
|
||||
.get(name)
|
||||
.ok_or_else(|| format!("file '{name}' listed in '{dsc_name}' has no checksum entry"))?;
|
||||
checksums.insert_entry(
|
||||
name,
|
||||
ChecksumEntry {
|
||||
size: p.size.unwrap_or(0),
|
||||
md5: p.md5.clone().unwrap_or_default(),
|
||||
sha1: p.sha1.clone().unwrap_or_default(),
|
||||
sha256: p.sha256.clone().unwrap_or_default(),
|
||||
// The `.dsc` records no SHA-512 (dpkg only writes
|
||||
// sha1/sha256 there).
|
||||
sha512: String::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -521,109 +432,11 @@ mod tests {
|
||||
assert!(!environment.contains("DEBIAN_FRONTEND"), "{environment}");
|
||||
}
|
||||
|
||||
/// A minimal previous-version `.dsc` with a 3-column Checksums-Sha1
|
||||
/// field, a 4-column Checksums-Sha256 line and a 3-column `Files`.
|
||||
/// Regression: the old code filled `names` from any line with a third
|
||||
/// column but `partials` only from exactly-3-column lines, so the
|
||||
/// "bogus" name landed in `names` alone and `&partials["bogus"]`
|
||||
/// panicked. It must produce a build error instead.
|
||||
#[test]
|
||||
fn dsc_four_column_checksum_line_errors_instead_of_panicking() {
|
||||
let dsc_name = "hello_1.0-1.dsc";
|
||||
let dsc = "\
|
||||
Format: 3.0 (native)
|
||||
Source: hello
|
||||
Binary: hello
|
||||
Architecture: any
|
||||
Version: 1.0-1
|
||||
Maintainer: A B <a@b.c>
|
||||
Checksums-Sha1:
|
||||
aaa111 12 hello_1.0.orig.tar.xz
|
||||
Checksums-Sha256:
|
||||
bbb222 12 bogus hello_1.0-1.debian.tar.xz
|
||||
Files:
|
||||
ddd333 12 hello_1.0.orig.tar.xz
|
||||
";
|
||||
let base = tempfile::tempdir().expect("tempdir");
|
||||
std::fs::write(base.path().join(dsc_name), dsc).expect("write dsc");
|
||||
|
||||
let ctx = Arc::new(
|
||||
crate::context::Context::new(crate::context::ContextConfig::Local).expect("context"),
|
||||
);
|
||||
let mut checksums = FileChecksums::new();
|
||||
let err = include_dsc_artifacts(&ctx, base.path(), dsc_name, &mut checksums)
|
||||
.expect_err("malformed Checksums-Sha256 line must fail the build");
|
||||
let err = err.to_string();
|
||||
assert!(err.contains("Checksums-Sha256"), "{err}");
|
||||
assert!(err.contains("hello_1.0-1.debian.tar.xz"), "{err}");
|
||||
}
|
||||
|
||||
/// Happy path: tarball entries are assembled from the Checksums fields
|
||||
/// (sha1/sha256) and merged with the legacy 5-column `Files` md5, in
|
||||
/// Checksums-Sha1 order, with the `.dsc` itself hashed fresh first.
|
||||
#[test]
|
||||
fn include_dsc_artifacts_merges_legacy_files_layout() {
|
||||
let dsc_name = "hello_1.0-1.dsc";
|
||||
let tarball = "hello_1.0.orig.tar.xz";
|
||||
let dsc = "\
|
||||
Format: 3.0 (quilt)
|
||||
Source: hello
|
||||
Binary: hello
|
||||
Architecture: any
|
||||
Version: 1.0-1
|
||||
Maintainer: A B <a@b.c>
|
||||
Checksums-Sha1:
|
||||
aaa111 12 hello_1.0.orig.tar.xz
|
||||
Checksums-Sha256:
|
||||
bbb222 12 hello_1.0.orig.tar.xz
|
||||
Files:
|
||||
ddd333 12 devel optional hello_1.0.orig.tar.xz
|
||||
";
|
||||
let base = tempfile::tempdir().expect("tempdir");
|
||||
std::fs::write(base.path().join(dsc_name), dsc).expect("write dsc");
|
||||
std::fs::write(base.path().join(tarball), "tarball bytes").expect("write tarball");
|
||||
|
||||
let ctx = Arc::new(
|
||||
crate::context::Context::new(crate::context::ContextConfig::Local).expect("context"),
|
||||
);
|
||||
let mut checksums = FileChecksums::new();
|
||||
include_dsc_artifacts(&ctx, base.path(), dsc_name, &mut checksums)
|
||||
.expect("valid dsc must parse");
|
||||
|
||||
let collected: Vec<(String, crate::debian::ChecksumEntry)> = checksums
|
||||
.iter()
|
||||
.map(|(k, e)| (k.clone(), e.clone()))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
collected
|
||||
.iter()
|
||||
.map(|(k, _)| k.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![dsc_name, tarball],
|
||||
".dsc first, then Checksums-Sha1 order"
|
||||
);
|
||||
|
||||
// The .dsc is hashed fresh from disk.
|
||||
let dsc_entry = &collected[0].1;
|
||||
assert_eq!(dsc_entry.size, dsc.len() as u64);
|
||||
assert_eq!(dsc_entry.md5.len(), 32);
|
||||
assert_eq!(dsc_entry.sha1.len(), 40);
|
||||
assert_eq!(dsc_entry.sha256.len(), 64);
|
||||
|
||||
// The tarball reuses the .dsc-recorded digests, including the
|
||||
// legacy 5-column `Files` md5 (section/priority skipped).
|
||||
let tar_entry = &collected[1].1;
|
||||
assert_eq!(tar_entry.size, 12);
|
||||
assert_eq!(tar_entry.md5, "ddd333");
|
||||
assert_eq!(tar_entry.sha1, "aaa111");
|
||||
assert_eq!(tar_entry.sha256, "bbb222");
|
||||
}
|
||||
|
||||
/// A binary-only (binNMU) build whose changelog cannot yield the
|
||||
/// previous entry (malformed second header, unbalanced parenthesis) must
|
||||
/// fail the metadata generation with a diagnostic naming the problem,
|
||||
/// instead of silently emitting a plain `Source:` `.changes` with no
|
||||
/// `Binary-Only-Changes` and no redistributed previous `.dsc`.
|
||||
/// `Binary-Only-Changes` and no previous-version reference.
|
||||
#[test]
|
||||
fn binary_only_prev_version_parse_failure_errors_instead_of_wrong_metadata() {
|
||||
let changelog = "\
|
||||
@@ -739,4 +552,104 @@ Description: test package
|
||||
#[cfg(unix)]
|
||||
assert!(err.contains("Permission denied"), "{err}");
|
||||
}
|
||||
|
||||
/// A binary-only (binNMU) build references the previous source version
|
||||
/// (`Source: pkg (prev)`, `Binary-Only-Changes`) but must NOT
|
||||
/// redistribute any source file: like dpkg-genchanges/genbuildinfo, the
|
||||
/// previous `.dsc` and its tarballs stay out of both documents even when
|
||||
/// they exist next to the artifacts.
|
||||
#[test]
|
||||
fn binary_only_metadata_references_previous_source_without_redistributing_it() {
|
||||
let changelog = "\
|
||||
hello (1.0-1+b1) unstable; urgency=medium, binary-only=yes
|
||||
|
||||
* Binary-only rebuild.
|
||||
|
||||
-- A B <a@b.c> Mon, 01 Jan 2024 00:00:00 +0000
|
||||
|
||||
hello (1.0-1) unstable; urgency=medium
|
||||
|
||||
* Initial release.
|
||||
|
||||
-- A B <a@b.c> Sun, 31 Dec 2023 00:00:00 +0000
|
||||
";
|
||||
let control = "\
|
||||
Source: hello
|
||||
Section: devel
|
||||
Priority: optional
|
||||
Maintainer: A B <a@b.c>
|
||||
|
||||
Package: hello
|
||||
Architecture: all
|
||||
Description: test package
|
||||
";
|
||||
let base = tempfile::tempdir().expect("tempdir");
|
||||
let tree = base.path().join("hello-1.0");
|
||||
std::fs::create_dir_all(tree.join("debian")).expect("mkdir tree");
|
||||
std::fs::write(tree.join("debian/changelog"), changelog).expect("write changelog");
|
||||
std::fs::write(tree.join("debian/control"), control).expect("write control");
|
||||
std::fs::write(
|
||||
tree.join("debian/files"),
|
||||
"hello_1.0-1+b1_all.deb devel optional\n",
|
||||
)
|
||||
.expect("write files");
|
||||
std::fs::write(base.path().join("hello_1.0-1+b1_all.deb"), "deb payload")
|
||||
.expect("write deb");
|
||||
|
||||
// The trap: the previous source artifacts sit right next to the
|
||||
// binaries, as they would after a source build. dpkg does not
|
||||
// redistribute them for a binary-only upload, and neither must we.
|
||||
std::fs::write(
|
||||
base.path().join("hello_1.0-1.dsc"),
|
||||
"Format: 3.0 (quilt)\nSource: hello\nBinary: hello\nArchitecture: any\nVersion: \
|
||||
1.0-1\nMaintainer: A B <a@b.c>\nChecksums-Sha1:\n aaa111 12 \
|
||||
hello_1.0.orig.tar.xz\n",
|
||||
)
|
||||
.expect("write previous dsc");
|
||||
std::fs::write(base.path().join("hello_1.0.orig.tar.xz"), "tarball bytes")
|
||||
.expect("write previous tarball");
|
||||
|
||||
let ctx = Arc::new(
|
||||
crate::context::Context::new(crate::context::ContextConfig::Local).expect("context"),
|
||||
);
|
||||
let opts = BinaryMetadataOptions {
|
||||
profiles: Vec::new(),
|
||||
vendor: "debian".to_string(),
|
||||
exported_env: BTreeMap::new(),
|
||||
build_arch: "amd64".to_string(),
|
||||
host_arch: "amd64".to_string(),
|
||||
};
|
||||
let (buildinfo_path, changes_path) =
|
||||
generate_binary_metadata(&ctx, &tree, base.path(), &opts)
|
||||
.expect("binNMU metadata generation must succeed");
|
||||
|
||||
let changes = std::fs::read_to_string(&changes_path).expect("read changes");
|
||||
let buildinfo = std::fs::read_to_string(&buildinfo_path).expect("read buildinfo");
|
||||
|
||||
// The previous version is referenced textually.
|
||||
assert!(
|
||||
changes.contains("Source: hello (1.0-1)"),
|
||||
"changes must reference the previous version: {changes}"
|
||||
);
|
||||
assert!(
|
||||
buildinfo.contains("Binary-Only-Changes"),
|
||||
"buildinfo must record the binary-only entry: {buildinfo}"
|
||||
);
|
||||
// ... but no source file is distributed, on either side.
|
||||
for (doc, text) in [("changes", &changes), ("buildinfo", &buildinfo)] {
|
||||
assert!(
|
||||
!text.contains("hello_1.0-1.dsc"),
|
||||
"{doc} must not redistribute the previous .dsc: {text}"
|
||||
);
|
||||
assert!(
|
||||
!text.contains("hello_1.0.orig.tar.xz"),
|
||||
"{doc} must not redistribute the previous tarball: {text}"
|
||||
);
|
||||
}
|
||||
// The distributed set is exactly the binary artifacts + buildinfo.
|
||||
assert!(
|
||||
changes.contains("hello_1.0-1+b1_all.deb") && changes.contains(".buildinfo"),
|
||||
"changes must distribute the deb and the buildinfo: {changes}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,6 +65,9 @@ pub struct ChangesInput {
|
||||
pub source: String,
|
||||
/// Sorted binary package names with artifacts (empty for source-only).
|
||||
pub binaries: Vec<String>,
|
||||
/// Whether the changelog entry is a binary-only (binNMU) upload
|
||||
/// (`Binary-Only: yes` field).
|
||||
pub binary_only: bool,
|
||||
/// Active build profiles (`Built-For-Profiles`); omitted when empty.
|
||||
pub built_for_profiles: Vec<String>,
|
||||
/// `Architecture` field value in encounter order (e.g. `source`,
|
||||
@@ -142,6 +145,9 @@ pub fn render_changes(input: &ChangesInput) -> Paragraph {
|
||||
let joined = input.binaries.join(" ");
|
||||
p.set("Binary", &wrap_long(&joined));
|
||||
}
|
||||
if input.binary_only {
|
||||
p.set("Binary-Only", "yes");
|
||||
}
|
||||
if !input.built_for_profiles.is_empty() {
|
||||
p.set("Built-For-Profiles", &input.built_for_profiles.join(" "));
|
||||
}
|
||||
@@ -329,6 +335,7 @@ mod tests {
|
||||
date: "Sat, 22 Aug 2026 10:00:00 +0000".to_string(),
|
||||
source: "pkg".to_string(),
|
||||
binaries: vec![],
|
||||
binary_only: false,
|
||||
built_for_profiles: vec![],
|
||||
architecture: "source".to_string(),
|
||||
version: "1.0".to_string(),
|
||||
|
||||
+197
-87
@@ -224,7 +224,8 @@ fn retry_after_revendor(
|
||||
/// Run the full native source-build pipeline in `cwd`.
|
||||
///
|
||||
/// Steps (mirroring `dpkg-buildpackage -S -I -i -nc -d`):
|
||||
/// 1. sanity checks and metadata resolution (changelog, control),
|
||||
/// 1. sanity checks and metadata resolution (changelog, control); a
|
||||
/// `binary-only=yes` changelog entry is refused, like `dpkg-source -b`,
|
||||
/// 2. environment setup (`SOURCE_DATE_EPOCH`, `DEB_BUILD_OPTIONS`, arch vars),
|
||||
/// 3. signing decision (key discovery, UNRELEASED handling),
|
||||
/// 4. `dpkg-source --before-build` then `dpkg-source -b`,
|
||||
@@ -281,33 +282,25 @@ pub fn run_source_build(
|
||||
let mut entries = crate::debian::changelog::parse_changelog_entries(&changelog_path, Some(2))?;
|
||||
let entry = entries.remove(0);
|
||||
let previous_entry = entries.into_iter().next();
|
||||
|
||||
// A binary-only (binNMU) changelog entry is a binary publication whose
|
||||
// source is already in the archive: like dpkg-source, refuse to build
|
||||
// source for it instead of producing binNMU-style source metadata.
|
||||
if entry.binary_only {
|
||||
return Err(
|
||||
"cannot build source for a binary-only publication: the changelog \
|
||||
entry sets binary-only=yes (dpkg-source refuses it too)"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
|
||||
let ctrl = ControlInfo::parse(&control_path)?;
|
||||
|
||||
if let Some(u) = &ui {
|
||||
u.set_build_target(&entry.source, &entry.version.full(), &entry.distribution);
|
||||
}
|
||||
|
||||
// binNMU builds reference the *previous* (source) version in their
|
||||
// artifact metadata, like dpkg-genchanges/genbuildinfo do.
|
||||
let previous_version = if entry.binary_only {
|
||||
previous_entry.as_ref().map(|e| e.version.full())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let source_display = if entry.binary_only {
|
||||
match previous_version.as_deref() {
|
||||
Some(prev) => format!("{} ({})", entry.source, prev),
|
||||
None => entry.source.clone(),
|
||||
}
|
||||
} else {
|
||||
entry.source.clone()
|
||||
};
|
||||
let binary_only_changes = entry.binary_only.then(|| {
|
||||
format!(
|
||||
"{}\n\n -- {} <{}> {}",
|
||||
entry.changes_field, entry.maintainer_name, entry.maintainer_email, entry.date_raw
|
||||
)
|
||||
});
|
||||
let source_display = entry.source.clone();
|
||||
|
||||
let sversion = entry.version.no_epoch();
|
||||
let dsc_name = format!("{}_{}.dsc", entry.source, sversion);
|
||||
@@ -431,35 +424,16 @@ pub fn run_source_build(
|
||||
.into());
|
||||
}
|
||||
|
||||
// Binary-only uploads redistribute the *previous* source: metadata
|
||||
// references the previous version's .dsc (which must already exist in
|
||||
// the output directory), exactly like dpkg-genchanges/genbuildinfo.
|
||||
let ref_dsc_name = match previous_version.as_deref().filter(|_| entry.binary_only) {
|
||||
Some(prev) => {
|
||||
let prev_version = crate::debian::DebianVersion::parse(prev)?;
|
||||
let name = format!("{}_{}.dsc", entry.source, prev_version.no_epoch());
|
||||
if !parent.join(&name).exists() {
|
||||
return Err(format!(
|
||||
"binary-only build requires the previous source '{} \
|
||||
{}' to exist next to the package",
|
||||
entry.source, prev
|
||||
)
|
||||
.into());
|
||||
}
|
||||
name
|
||||
}
|
||||
None => dsc_name.clone(),
|
||||
};
|
||||
let ref_dsc_path = parent.join(&ref_dsc_name);
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// 6. .buildinfo generation (native dpkg-genbuildinfo equivalent)
|
||||
// ------------------------------------------------------------------
|
||||
if let Some(u) = &ui {
|
||||
u.progress_message("Generating .buildinfo");
|
||||
}
|
||||
let mut checksums = FileChecksums::new();
|
||||
checksums.add_file_as(&ref_dsc_path, &ref_dsc_name)?;
|
||||
// What the .buildinfo itself records: like dpkg-genbuildinfo, only the
|
||||
// referenced .dsc — not the tarballs, and never the buildinfo itself.
|
||||
let mut buildinfo_checksums = FileChecksums::new();
|
||||
buildinfo_checksums.add_file_as(&dsc_path, &dsc_name)?;
|
||||
|
||||
let status_path = PathBuf::from("/var/lib/dpkg/status");
|
||||
let bd_fields = [ctrl.source.get("Build-Depends").unwrap_or("")];
|
||||
@@ -472,7 +446,9 @@ pub fn run_source_build(
|
||||
binaries: Vec::new(), // source-only build
|
||||
architecture: "source".to_string(),
|
||||
version: entry.version.full(),
|
||||
binary_only_changes: binary_only_changes.clone(),
|
||||
// A binary-only entry never reaches a source build (refused
|
||||
// above), so the buildinfo never carries Binary-Only-Changes.
|
||||
binary_only_changes: None,
|
||||
build_origin: vendor.clone(),
|
||||
build_architecture: arch_vars
|
||||
.get("DEB_BUILD_ARCH")
|
||||
@@ -484,7 +460,7 @@ pub fn run_source_build(
|
||||
environment: environment.clone(),
|
||||
})
|
||||
};
|
||||
buildinfo::save_buildinfo(&buildinfo_path, &render_buildinfo_doc(&checksums))?;
|
||||
buildinfo::save_buildinfo(&buildinfo_path, &render_buildinfo_doc(&buildinfo_checksums))?;
|
||||
|
||||
// Register the .buildinfo in debian/files (as dpkg-genbuildinfo does).
|
||||
let files_path = cwd.join("debian/files");
|
||||
@@ -506,17 +482,21 @@ pub fn run_source_build(
|
||||
if let Some(u) = &ui {
|
||||
u.progress_message("Generating .changes");
|
||||
}
|
||||
// What the .changes distributes: the .dsc (with its recorded digests),
|
||||
// the tarballs listed in it (below), and the .buildinfo (last, as
|
||||
// dpkg-genchanges does when it consumes debian/files).
|
||||
let mut checksums = buildinfo_checksums.clone();
|
||||
// Pull the tarball checksums out of the referenced .dsc so they are
|
||||
// distributed through the .changes like dpkg-genchanges does, in the
|
||||
// order the .dsc itself lists them.
|
||||
let dsc_content = std::fs::read_to_string(&ref_dsc_path)
|
||||
.map_err(|e| format!("cannot read '{}': {}", ref_dsc_path.display(), e))?;
|
||||
let dsc_content = std::fs::read_to_string(&dsc_path)
|
||||
.map_err(|e| format!("cannot read '{}': {}", dsc_path.display(), e))?;
|
||||
let dsc_para = parse_paragraphs(crate::debian::control::strip_clearsigned_armour(
|
||||
&dsc_content,
|
||||
))
|
||||
.into_iter()
|
||||
.next()
|
||||
.ok_or_else(|| format!("'{}' is empty", ref_dsc_path.display()))?;
|
||||
.ok_or_else(|| format!("'{}' is empty", dsc_path.display()))?;
|
||||
|
||||
// Whether the upload redistributes the upstream tarballs (dpkg
|
||||
// -sa/-si/-sd source styles). Stripping only applies to a split source
|
||||
@@ -536,7 +516,7 @@ pub fn run_source_build(
|
||||
continue;
|
||||
};
|
||||
for cl in parse_checksum_field(field, value)
|
||||
.map_err(|e| format!("cannot parse '{}': {e}", ref_dsc_path.display()))?
|
||||
.map_err(|e| format!("cannot parse '{}': {e}", dsc_path.display()))?
|
||||
{
|
||||
if !dsc_files.contains_key(&cl.name) {
|
||||
dsc_file_names.push(cl.name.clone());
|
||||
@@ -576,7 +556,7 @@ pub fn run_source_build(
|
||||
};
|
||||
|
||||
for name in &dsc_file_names {
|
||||
if name == &ref_dsc_name {
|
||||
if name == &dsc_name {
|
||||
continue; // already computed directly above
|
||||
}
|
||||
if is_stripped(name) {
|
||||
@@ -594,7 +574,7 @@ pub fn run_source_build(
|
||||
let partial = dsc_files.get(name).ok_or_else(|| {
|
||||
format!(
|
||||
"file '{name}' listed in '{}' has no checksum entry",
|
||||
ref_dsc_path.display()
|
||||
dsc_path.display()
|
||||
)
|
||||
})?;
|
||||
checksums.insert_entry(
|
||||
@@ -621,13 +601,9 @@ pub fn run_source_build(
|
||||
// dsc and tarballs use the source stanza defaults (not persisted into
|
||||
// debian/files, matching dpkg).
|
||||
let mut changes_files = files_list.clone();
|
||||
changes_files.add(FilesEntry::new(
|
||||
&ref_dsc_name,
|
||||
ctrl.section(),
|
||||
ctrl.priority(),
|
||||
));
|
||||
changes_files.add(FilesEntry::new(&dsc_name, ctrl.section(), ctrl.priority()));
|
||||
for name in &dsc_file_names {
|
||||
if name != &ref_dsc_name && !is_stripped(name) {
|
||||
if name != &dsc_name && !is_stripped(name) {
|
||||
changes_files.add(FilesEntry::new(name, ctrl.section(), ctrl.priority()));
|
||||
}
|
||||
}
|
||||
@@ -638,6 +614,7 @@ pub fn run_source_build(
|
||||
date: entry.date_raw.clone(),
|
||||
source: source_display.clone(),
|
||||
binaries: Vec::new(), // source-only upload
|
||||
binary_only: entry.binary_only,
|
||||
built_for_profiles: profiles.clone(),
|
||||
architecture: "source".to_string(),
|
||||
version: entry.version.full(),
|
||||
@@ -681,18 +658,18 @@ pub fn run_source_build(
|
||||
|
||||
log::info!("Signing {}", dsc_name);
|
||||
crate::utils::gpg::clearsign_file(&dsc_path, &keyid)?;
|
||||
// The freshly built .dsc changed: refresh its checksums inside the
|
||||
// .buildinfo. For binary-only builds the metadata references the
|
||||
// *previous* .dsc (untouched by this build), so there is nothing to
|
||||
// refresh.
|
||||
if !entry.binary_only {
|
||||
// The freshly built .dsc changed: refresh its digests in both
|
||||
// checksum sets.
|
||||
buildinfo_checksums.add_file_as(&dsc_path, &dsc_name)?;
|
||||
checksums.add_file_as(&dsc_path, &dsc_name)?;
|
||||
}
|
||||
buildinfo::save_buildinfo(&buildinfo_path, &render_buildinfo_doc(&checksums))?;
|
||||
// Re-render the .buildinfo from its own set (the .dsc only, like
|
||||
// dpkg-genbuildinfo): it must not list the tarballs or itself.
|
||||
buildinfo::save_buildinfo(&buildinfo_path, &render_buildinfo_doc(&buildinfo_checksums))?;
|
||||
|
||||
log::info!("Signing {}", buildinfo_name);
|
||||
crate::utils::gpg::clearsign_file(&buildinfo_path, &keyid)?;
|
||||
// Both .dsc and .buildinfo changed: refresh the .changes.
|
||||
// Both .dsc and .buildinfo changed: refresh the .changes with the
|
||||
// signed buildinfo's fresh digests.
|
||||
checksums.add_file_as(&buildinfo_path, &buildinfo_name)?;
|
||||
changes::save_changes(&changes_path, &render_changes_doc(&checksums))?;
|
||||
|
||||
@@ -1040,6 +1017,35 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::debian::DebianVersion;
|
||||
|
||||
/// A binary-only (binNMU) changelog entry is a binary publication: like
|
||||
/// `dpkg-source -b`, the source build must refuse it outright instead of
|
||||
/// producing binNMU-style source metadata referencing the previous
|
||||
/// version's `.dsc`.
|
||||
#[test]
|
||||
fn source_build_refuses_binary_only_changelog() {
|
||||
let base = tempfile::tempdir().expect("tempdir");
|
||||
let tree = base.path().join("hello-1.0");
|
||||
std::fs::create_dir_all(tree.join("debian")).expect("mkdir tree");
|
||||
std::fs::write(
|
||||
tree.join("debian/changelog"),
|
||||
"hello (1.0-1+b1) unstable; urgency=medium, binary-only=yes\n\n \
|
||||
* Binary-only rebuild.\n\n -- A B <a@b.c> Mon, 01 Jan 2024 00:00:00 +0000\n",
|
||||
)
|
||||
.expect("write changelog");
|
||||
std::fs::write(
|
||||
tree.join("debian/control"),
|
||||
"Source: hello\nMaintainer: A B <a@b.c>\n",
|
||||
)
|
||||
.expect("write control");
|
||||
std::fs::write(tree.join("debian/rules"), "#!/usr/bin/make -f\n").expect("write rules");
|
||||
|
||||
let err = run_source_build(&tree, &SourceBuildOptions::default(), None)
|
||||
.expect_err("binary-only entries must not build a source package");
|
||||
let err = err.to_string();
|
||||
assert!(err.contains("binary-only"), "{err}");
|
||||
assert!(err.contains("source"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_checksum_defaults() {
|
||||
let p = PartialChecksum::default();
|
||||
@@ -1266,6 +1272,8 @@ mod differential_tests {
|
||||
extra_source_fields: &'static [(&'static str, &'static str)],
|
||||
/// Version of the previous changelog entry, when the fixture has one.
|
||||
previous_version: Option<&'static str>,
|
||||
/// Mark the newest changelog entry `binary-only=yes` (binNMU).
|
||||
binary_only_marker: bool,
|
||||
}
|
||||
|
||||
impl FixtureSpec {
|
||||
@@ -1283,6 +1291,7 @@ mod differential_tests {
|
||||
patches: &[],
|
||||
extra_source_fields: &[],
|
||||
previous_version: None,
|
||||
binary_only_marker: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1295,9 +1304,14 @@ mod differential_tests {
|
||||
}
|
||||
|
||||
fn changelog(&self) -> String {
|
||||
let params = if self.binary_only_marker {
|
||||
format!("urgency={}, binary-only=yes", self.urgency)
|
||||
} else {
|
||||
format!("urgency={}", self.urgency)
|
||||
};
|
||||
let mut out = format!(
|
||||
"{} ({}) {}; urgency={}\n\n",
|
||||
self.name, self.version, self.distribution, self.urgency
|
||||
"{} ({}) {}; {}\n\n",
|
||||
self.name, self.version, self.distribution, params
|
||||
);
|
||||
for line in self.body {
|
||||
out.push_str(" * ");
|
||||
@@ -1863,23 +1877,50 @@ Provides: virtual-thing (= 2.0), plain-virtual
|
||||
#[test]
|
||||
fn diff_binary_build_metadata() {
|
||||
const NAME: &str = "pkh-diff-m";
|
||||
let control = format!(
|
||||
"Source: {NAME}\nSection: utils\nPriority: optional\nMaintainer: {MAINTAINER}\nBuild-Depends: libc6\n\n\
|
||||
Package: {NAME}\nArchitecture: any\nDescription: test package main\n long description\n\n\
|
||||
Package: {NAME}-u\nPackage-Type: udeb\nArchitecture: all\nDescription: test udeb\n short\n"
|
||||
);
|
||||
let changelog = format!(
|
||||
"{NAME} (1.0-1) unstable; urgency=medium\n\n * Binary build test.\n\n -- {MAINTAINER} {DATE}\n"
|
||||
);
|
||||
let rules = format!(
|
||||
"#!/usr/bin/make -f\nV = $(shell dpkg-parsechangelog -S Version)\nA = $(shell dpkg-architecture -qDEB_HOST_ARCH)\n\nbuild:\n\tmkdir -p debian/tmp/usr/bin\n\tprintf '#!/bin/sh\\necho hi\\n' > debian/tmp/usr/bin/hello\n\tchmod 755 debian/tmp/usr/bin/hello\n\ttouch $@\n\nbinary: build\n\trm -rf debian/{NAME} debian/{NAME}-u\n\tmkdir -p debian/{NAME}/usr/bin debian/{NAME}/DEBIAN\n\tcp -r debian/tmp/. debian/{NAME}/\n\tdpkg-gencontrol -p{NAME} -Pdebian/{NAME}\n\tdpkg-deb --build debian/{NAME} ..\n\tmkdir -p debian/{NAME}-u/usr/share debian/{NAME}-u/DEBIAN\n\techo data > debian/{NAME}-u/usr/share/data.txt\n\tdpkg-gencontrol -p{NAME}-u -Pdebian/{NAME}-u\n\tdpkg-deb --build debian/{NAME}-u ..\n\tmv ../{NAME}-u_$(V)_all.deb ../{NAME}-u_$(V)_all.udeb\n\nclean:\n\trm -rf debian/tmp debian/{NAME} debian/{NAME}-u build-stamp debian/files debian/*.substvars\n\n.PHONY: build binary clean\n"
|
||||
);
|
||||
diff_binary_metadata_case(NAME, &changelog, "1.0-1", false);
|
||||
}
|
||||
|
||||
/// debian/control shared by the binary-metadata differential cases: one
|
||||
/// arch:any deb and one arch:all udeb.
|
||||
fn binary_test_control(name: &str) -> String {
|
||||
format!(
|
||||
"Source: {name}\nSection: utils\nPriority: optional\nMaintainer: {MAINTAINER}\nBuild-Depends: libc6\n\n\
|
||||
Package: {name}\nArchitecture: any\nDescription: test package main\n long description\n\n\
|
||||
Package: {name}-u\nPackage-Type: udeb\nArchitecture: all\nDescription: test udeb\n short\n"
|
||||
)
|
||||
}
|
||||
|
||||
/// debian/rules driving dpkg-gencontrol/dpkg-deb directly (no debhelper).
|
||||
fn binary_test_rules(name: &str) -> String {
|
||||
format!(
|
||||
"#!/usr/bin/make -f\nV = $(shell dpkg-parsechangelog -S Version)\nA = $(shell dpkg-architecture -qDEB_HOST_ARCH)\n\nbuild:\n\tmkdir -p debian/tmp/usr/bin\n\tprintf '#!/bin/sh\\necho hi\\n' > debian/tmp/usr/bin/hello\n\tchmod 755 debian/tmp/usr/bin/hello\n\ttouch $@\n\nbinary: build\n\trm -rf debian/{name} debian/{name}-u\n\tmkdir -p debian/{name}/usr/bin debian/{name}/DEBIAN\n\tcp -r debian/tmp/. debian/{name}/\n\tdpkg-gencontrol -p{name} -Pdebian/{name}\n\tdpkg-deb --build debian/{name} ..\n\tmkdir -p debian/{name}-u/usr/share debian/{name}-u/DEBIAN\n\techo data > debian/{name}-u/usr/share/data.txt\n\tdpkg-gencontrol -p{name}-u -Pdebian/{name}-u\n\tdpkg-deb --build debian/{name}-u ..\n\tmv ../{name}-u_$(V)_all.deb ../{name}-u_$(V)_all.udeb\n\nclean:\n\trm -rf debian/tmp debian/{name} debian/{name}-u build-stamp debian/files debian/*.substvars\n\n.PHONY: build binary clean\n"
|
||||
)
|
||||
}
|
||||
|
||||
/// Both-sides binary metadata comparison for one changelog: golden
|
||||
/// `dpkg-buildpackage -b` against the pkh deb flow (rules targets with a
|
||||
/// dpkg-buildpackage-like environment) plus native metadata generation.
|
||||
/// `artifact_version` is the full version the artifacts are named after;
|
||||
/// `with_prev_source` additionally places the previous version's `.dsc`
|
||||
/// and tarball next to the tree on both sides (the binNMU trap: they must
|
||||
/// not be redistributed).
|
||||
fn diff_binary_metadata_case(
|
||||
name: &str,
|
||||
changelog: &str,
|
||||
artifact_version: &str,
|
||||
with_prev_source: bool,
|
||||
) {
|
||||
let control = binary_test_control(name);
|
||||
let rules = binary_test_rules(name);
|
||||
|
||||
let write_tree = |root: &Path| {
|
||||
fs::create_dir_all(root.join(format!("{NAME}/debian/source"))).expect("mkdir tree");
|
||||
let tree = root.join(NAME);
|
||||
fs::create_dir_all(root.join(format!("{name}/debian/source"))).expect("mkdir tree");
|
||||
let tree = root.join(name);
|
||||
fs::write(tree.join("debian/control"), &control).expect("write control");
|
||||
fs::write(tree.join("debian/changelog"), &changelog).expect("write changelog");
|
||||
fs::write(tree.join("debian/changelog"), changelog).expect("write changelog");
|
||||
fs::write(tree.join("debian/source/format"), "3.0 (native)\n").expect("write format");
|
||||
fs::write(tree.join("debian/rules"), &rules).expect("write rules");
|
||||
#[cfg(unix)]
|
||||
@@ -1900,6 +1941,20 @@ Provides: virtual-thing (= 2.0), plain-virtual
|
||||
let golden_tree = write_tree(&golden_root);
|
||||
let ours_tree = write_tree(&ours_root);
|
||||
|
||||
if with_prev_source {
|
||||
for root in [&golden_root, &ours_root] {
|
||||
let dsc = format!(
|
||||
"Format: 3.0 (native)\nSource: {name}\nBinary: {name}\nArchitecture: all\n\
|
||||
Version: 1.0-1\nMaintainer: {MAINTAINER}\nChecksums-Sha1:\n aaa111 12 \
|
||||
{name}_1.0.tar.xz\nChecksums-Sha256:\n bbb222 12 {name}_1.0.tar.xz\nFiles:\n \
|
||||
ddd333 12 utils optional {name}_1.0.tar.xz\n"
|
||||
);
|
||||
fs::write(root.join(format!("{name}_1.0-1.dsc")), dsc).expect("write previous dsc");
|
||||
fs::write(root.join(format!("{name}_1.0.tar.xz")), "tarball byte")
|
||||
.expect("write previous tarball");
|
||||
}
|
||||
}
|
||||
|
||||
// Golden side: real dpkg-buildpackage binary build.
|
||||
let status = crate::test_support::run_logged(
|
||||
Command::new("dpkg-buildpackage")
|
||||
@@ -1915,7 +1970,7 @@ Provides: virtual-thing (= 2.0), plain-virtual
|
||||
// rules targets directly by default (missing Rules-Requires-Root is
|
||||
// treated as 'no'), so no fakeroot wrapper here either.
|
||||
let entry =
|
||||
crate::debian::parse_changelog_entry_from_str(&changelog).expect("parse changelog");
|
||||
crate::debian::parse_changelog_entry_from_str(changelog).expect("parse changelog");
|
||||
let vendor = env::current_vendor();
|
||||
let profiles = env::resolve_build_profiles(&[], &vendor);
|
||||
let parallel = env::num_parallel();
|
||||
@@ -1959,15 +2014,32 @@ Provides: virtual-thing (= 2.0), plain-virtual
|
||||
|
||||
// Compare artifacts.
|
||||
assert_changes_equivalent(
|
||||
&golden_root.join(format!("{NAME}_1.0-1_amd64.changes")),
|
||||
&ours_root.join(format!("{NAME}_1.0-1_amd64.changes")),
|
||||
&golden_root.join(format!("{name}_{artifact_version}_amd64.changes")),
|
||||
&ours_root.join(format!("{name}_{artifact_version}_amd64.changes")),
|
||||
);
|
||||
assert_buildinfo_equivalent(
|
||||
&golden_root.join(format!("{NAME}_1.0-1_amd64.buildinfo")),
|
||||
&ours_root.join(format!("{NAME}_1.0-1_amd64.buildinfo")),
|
||||
&golden_root.join(format!("{name}_{artifact_version}_amd64.buildinfo")),
|
||||
&ours_root.join(format!("{name}_{artifact_version}_amd64.buildinfo")),
|
||||
);
|
||||
}
|
||||
|
||||
/// Differential check of the binary-only (binNMU) metadata against real
|
||||
/// `dpkg-buildpackage -b`: with the previous source artifacts sitting
|
||||
/// next to the tree (as after a source build), the `.changes` must
|
||||
/// distribute only the binaries and the `.buildinfo`, both documents
|
||||
/// referencing the previous version textually only. Regression guard for
|
||||
/// the previous-source redistribution pkh used to emit.
|
||||
#[test]
|
||||
fn diff_binmu_binary_metadata() {
|
||||
const NAME: &str = "pkh-diff-r";
|
||||
let changelog = format!(
|
||||
"{NAME} (1.0-1+b1) unstable; urgency=medium, binary-only=yes\n\n * Binary-only \
|
||||
rebuild.\n\n -- {MAINTAINER} {DATE}\n\n{NAME} (1.0-1) unstable; urgency=medium\n\n \
|
||||
* Initial release.\n\n -- {MAINTAINER} {DATE}\n"
|
||||
);
|
||||
diff_binary_metadata_case(NAME, &changelog, "1.0-1+b1", true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diff_native_minimal() {
|
||||
differential_case(&FixtureSpec::new("pkh-diff-a", "1.0-1", "unstable"));
|
||||
@@ -2039,8 +2111,10 @@ Provides: virtual-thing (= 2.0), plain-virtual
|
||||
spec.body = &["* Binary-only rebuild."];
|
||||
spec.previous_version = Some("1.0-1");
|
||||
|
||||
// Binary-only metadata references the previous version's .dsc, which
|
||||
// must already exist next to the package tree.
|
||||
// A binNMU-style version number (+b1) with a previous entry, but
|
||||
// WITHOUT the binary-only marker: this is a plain source build, and
|
||||
// the sibling previous-version .dsc (as left by an earlier source
|
||||
// build) must not change either side's output.
|
||||
let base = tempfile::tempdir().expect("tempdir");
|
||||
let tree = write_fixture(base.path(), &spec);
|
||||
let prev_dsc = format!(
|
||||
@@ -2055,6 +2129,42 @@ Provides: virtual-thing (= 2.0), plain-virtual
|
||||
differential_on_tree(&tree, &SourceBuildOptions::default());
|
||||
}
|
||||
|
||||
/// Both implementations must refuse a source build of a changelog entry
|
||||
/// marked `binary-only=yes`: dpkg-source errors out, and pkh must refuse
|
||||
/// the same way instead of producing binNMU-style source metadata
|
||||
/// referencing the previous version.
|
||||
#[test]
|
||||
fn diff_source_build_rejects_binary_only_marker() {
|
||||
let mut spec = FixtureSpec::new("pkh-diff-s", "1.0-1+b1", "unstable");
|
||||
spec.body = &["* Binary-only rebuild."];
|
||||
spec.previous_version = Some("1.0-1");
|
||||
spec.binary_only_marker = true;
|
||||
let base = tempfile::tempdir().expect("tempdir");
|
||||
let tree = write_fixture(base.path(), &spec);
|
||||
|
||||
// Golden side: real dpkg refuses.
|
||||
let status = crate::test_support::run_logged(
|
||||
Command::new("dpkg-buildpackage").current_dir(&tree).args([
|
||||
"-S",
|
||||
"-I",
|
||||
"-i",
|
||||
"-nc",
|
||||
"-d",
|
||||
"--no-sign",
|
||||
]),
|
||||
)
|
||||
.expect("run dpkg-buildpackage (is dpkg-dev installed?)");
|
||||
assert!(
|
||||
!status.success(),
|
||||
"dpkg-buildpackage -S must refuse a binary-only changelog entry"
|
||||
);
|
||||
|
||||
// Ours: the native pipeline refuses likewise.
|
||||
let err = run_source_build(&tree, &SourceBuildOptions::default(), None)
|
||||
.expect_err("native source build must refuse a binary-only entry");
|
||||
assert!(err.to_string().contains("binary-only"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diff_closes_bugs() {
|
||||
let mut spec = FixtureSpec::new("pkh-diff-j", "2.0-1", "unstable");
|
||||
|
||||
+260
-42
@@ -31,6 +31,20 @@ const LP_USER_KEY: &str = "lp.user";
|
||||
/// Base URL of the Launchpad REST API
|
||||
const API_BASE: &str = "https://api.launchpad.net/1.0";
|
||||
|
||||
/// Page size (`ws.size`) asked from Launchpad collections. Launchpad
|
||||
/// truncates collection answers at 75 entries by default and rejects
|
||||
/// `ws.size` above 300 (both verified against the live API); 100 sits
|
||||
/// comfortably under the cap while keeping multi-page walks rare.
|
||||
const WS_PAGE_SIZE: u32 = 100;
|
||||
|
||||
/// Hard cap on the pages followed while walking a `getPublishedSources`
|
||||
/// collection: 20 pages x 100 entries = 2000 currently published entries
|
||||
/// for one source name. The query only counts `Published` entries of live
|
||||
/// series/pockets, so real histories are a handful of entries; a walk
|
||||
/// reaching the cap means the API is misbehaving (an endless next-link
|
||||
/// chain), not that the history is genuinely huge.
|
||||
const MAX_COLLECTION_PAGES: u32 = 20;
|
||||
|
||||
/// The Launchpad username configured in git: the repository-local
|
||||
/// configuration wins over the global one, like git's own precedence.
|
||||
/// `None` when no git repository is found or the key is unset.
|
||||
@@ -178,20 +192,21 @@ fn percent_encode(value: &str) -> String {
|
||||
encoded
|
||||
}
|
||||
|
||||
/// URL of the `getPublishedSources` API call listing the currently
|
||||
/// `Published` source packages named `source_name` in the PPA `user/name`:
|
||||
/// `exact_match` avoids Launchpad's default case-insensitive substring
|
||||
/// matching, which would return unrelated sources (`data` matching
|
||||
/// `datatables`).
|
||||
/// URL of the first page of the `getPublishedSources` API call listing the
|
||||
/// currently `Published` source packages named `source_name` in the PPA
|
||||
/// `user/name`: `exact_match` avoids Launchpad's default case-insensitive
|
||||
/// substring matching, which would return unrelated sources (`data` matching
|
||||
/// `datatables`). Further pages are reached through the answer's
|
||||
/// `next_collection_link`, not by hand-building URLs.
|
||||
fn published_sources_url(user: &str, ppa: &str, source_name: &str) -> String {
|
||||
format!(
|
||||
"{}?ws.op=getPublishedSources&source_name={}&exact_match=true&status=Published",
|
||||
"{}?ws.op=getPublishedSources&source_name={}&exact_match=true&status=Published&ws.size={WS_PAGE_SIZE}",
|
||||
archive_url(user, ppa),
|
||||
percent_encode(source_name)
|
||||
)
|
||||
}
|
||||
|
||||
/// One entry of a `getPublishedSources` answer: the subset of the source
|
||||
/// One page of a `getPublishedSources` answer: the subset of the source
|
||||
/// package publishing history the superseded-upload check needs (the live
|
||||
/// answer carries many more fields, ignored by serde)
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -200,42 +215,52 @@ struct PublishedSource {
|
||||
source_package_version: String,
|
||||
}
|
||||
|
||||
/// The `getPublishedSources` collection answer
|
||||
/// One page of the `getPublishedSources` collection answer
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct PublishedSources {
|
||||
/// The currently published source packages matching the query
|
||||
/// The currently published source packages matching the query, on this
|
||||
/// page only
|
||||
#[serde(default)]
|
||||
entries: Vec<PublishedSource>,
|
||||
/// URL of the next page, present only when the collection was
|
||||
/// truncated (Launchpad answers carry it as a plain JSON string)
|
||||
next_collection_link: Option<String>,
|
||||
}
|
||||
|
||||
/// Every version of `source_name` currently `Published` in the PPA
|
||||
/// `user/name` (same `user/ppa_name` format as `pkh put --ppa`), in API
|
||||
/// order. Empty when the source was never published there — a 200 answer
|
||||
/// with zero entries, the normal first-upload case.
|
||||
pub async fn published_versions(
|
||||
ppa: &str,
|
||||
source_name: &str,
|
||||
) -> Result<Vec<String>, Box<dyn Error>> {
|
||||
let (user, name) = split_ppa(ppa)?;
|
||||
let client = crate::distro_info::http_client();
|
||||
/// Parse one page of a `getPublishedSources` collection into the versions
|
||||
/// it carries plus the link to the next page (`None` on the last one): the
|
||||
/// pagination decision, factored out of the HTTP walk so it can be tested
|
||||
/// without a server.
|
||||
fn parse_collection_page(body: &str) -> Result<(Vec<String>, Option<String>), serde_json::Error> {
|
||||
let sources: PublishedSources = serde_json::from_str(body)?;
|
||||
Ok((
|
||||
sources
|
||||
.entries
|
||||
.into_iter()
|
||||
.map(|entry| entry.source_package_version)
|
||||
.collect(),
|
||||
sources.next_collection_link,
|
||||
))
|
||||
}
|
||||
|
||||
/// GET one page of a collection, mapping the API statuses to the same
|
||||
/// errors as the other Launchpad calls (404 means the PPA does not exist).
|
||||
/// Returns the response body for [`parse_collection_page`].
|
||||
async fn fetch_collection_page(
|
||||
client: &reqwest::Client,
|
||||
url: &str,
|
||||
ppa: &str,
|
||||
) -> Result<String, Box<dyn Error>> {
|
||||
let response = client
|
||||
.get(published_sources_url(&user, &name, source_name))
|
||||
.get(url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("cannot reach the Launchpad API: {e}"))?;
|
||||
match response.status() {
|
||||
reqwest::StatusCode::OK => {
|
||||
let sources: PublishedSources = response
|
||||
.json()
|
||||
reqwest::StatusCode::OK => response
|
||||
.text()
|
||||
.await
|
||||
.map_err(|e| format!("cannot parse the Launchpad API response for '{ppa}': {e}"))?;
|
||||
Ok(sources
|
||||
.entries
|
||||
.into_iter()
|
||||
.map(|entry| entry.source_package_version)
|
||||
.collect())
|
||||
}
|
||||
.map_err(|e| format!("cannot read the Launchpad API response for '{ppa}': {e}").into()),
|
||||
reqwest::StatusCode::NOT_FOUND => {
|
||||
Err(format!("PPA '{ppa}' does not exist: create it on launchpad.net first").into())
|
||||
}
|
||||
@@ -243,6 +268,66 @@ pub async fn published_versions(
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk a `getPublishedSources` collection page by page: fetch the first
|
||||
/// page, then follow `next_collection_link` (the canonical Launchpad
|
||||
/// pagination) until a page comes without one, accumulating the versions of
|
||||
/// every page in order.
|
||||
///
|
||||
/// Exceeding [`MAX_COLLECTION_PAGES`] errors rather than returning the
|
||||
/// partial list: the result feeds `put`'s superseded-upload check, where a
|
||||
/// silently truncated list is exactly the bug pagination fixes — a
|
||||
/// superseded upload wrongly allowed through, to be rejected (or to
|
||||
/// silently supersede) in Launchpad's queue hours later. Every other
|
||||
/// failure mode of this check (network, HTTP status, parsing) aborts the
|
||||
/// upload too, and `put` fails before anything is written, so erring costs
|
||||
/// only a clear message.
|
||||
async fn walk_collection(
|
||||
client: &reqwest::Client,
|
||||
first_url: &str,
|
||||
ppa: &str,
|
||||
) -> Result<Vec<String>, Box<dyn Error>> {
|
||||
let mut versions = Vec::new();
|
||||
let mut url = first_url.to_string();
|
||||
for _page in 1..=MAX_COLLECTION_PAGES {
|
||||
let body = fetch_collection_page(client, &url, ppa).await?;
|
||||
let (mut page_versions, next) = parse_collection_page(&body)
|
||||
.map_err(|e| format!("cannot parse the Launchpad API response for '{ppa}': {e}"))?;
|
||||
versions.append(&mut page_versions);
|
||||
match next {
|
||||
Some(next) => url = next,
|
||||
None => return Ok(versions),
|
||||
}
|
||||
}
|
||||
|
||||
Err(format!(
|
||||
"the Launchpad API keeps paginating the published sources of '{ppa}' \
|
||||
after {MAX_COLLECTION_PAGES} pages: cannot run the superseded check \
|
||||
on a partial list"
|
||||
)
|
||||
.into())
|
||||
}
|
||||
|
||||
/// Every version of `source_name` currently `Published` in the PPA
|
||||
/// `user/name` (same `user/ppa_name` format as `pkh put --ppa`), in API
|
||||
/// order. Empty when the source was never published there — a 200 answer
|
||||
/// with zero entries, the normal first-upload case. Launchpad truncates
|
||||
/// collections per page, so the walk follows the API's `next_collection_link`
|
||||
/// until the collection is exhausted: a single page would miss the highest
|
||||
/// version of a source published in many series/pockets over time, and the
|
||||
/// superseded check would wrongly pass.
|
||||
pub async fn published_versions(
|
||||
ppa: &str,
|
||||
source_name: &str,
|
||||
) -> Result<Vec<String>, Box<dyn Error>> {
|
||||
let (user, name) = split_ppa(ppa)?;
|
||||
walk_collection(
|
||||
crate::distro_info::http_client(),
|
||||
&published_sources_url(&user, &name, source_name),
|
||||
ppa,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// PPA uploads only target Ubuntu series: fail before uploading when the
|
||||
/// changes' distribution is not a known series (typo) or a non-Ubuntu one —
|
||||
/// both are only rejected during queue processing otherwise
|
||||
@@ -372,16 +457,17 @@ mod tests {
|
||||
}
|
||||
|
||||
/// The query matches the verified live `getPublishedSources` call, with
|
||||
/// the source name percent-encoded
|
||||
/// the source name percent-encoded and an explicit page size (the API
|
||||
/// default of 75 entries would hide part of long publishing histories)
|
||||
#[test]
|
||||
fn published_sources_url_matches_launchpad_call() {
|
||||
assert_eq!(
|
||||
published_sources_url("vhaudiquet", "noctalia", "noctalia"),
|
||||
"https://api.launchpad.net/1.0/~vhaudiquet/+archive/ubuntu/noctalia?ws.op=getPublishedSources&source_name=noctalia&exact_match=true&status=Published"
|
||||
"https://api.launchpad.net/1.0/~vhaudiquet/+archive/ubuntu/noctalia?ws.op=getPublishedSources&source_name=noctalia&exact_match=true&status=Published&ws.size=100"
|
||||
);
|
||||
assert_eq!(
|
||||
published_sources_url("vhaudiquet", "noctalia", "g++"),
|
||||
"https://api.launchpad.net/1.0/~vhaudiquet/+archive/ubuntu/noctalia?ws.op=getPublishedSources&source_name=g%2B%2B&exact_match=true&status=Published"
|
||||
"https://api.launchpad.net/1.0/~vhaudiquet/+archive/ubuntu/noctalia?ws.op=getPublishedSources&source_name=g%2B%2B&exact_match=true&status=Published&ws.size=100"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -411,21 +497,153 @@ mod tests {
|
||||
]
|
||||
}"#;
|
||||
|
||||
let sources: PublishedSources = serde_json::from_str(json).unwrap();
|
||||
let versions: Vec<&str> = sources
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| entry.source_package_version.as_str())
|
||||
.collect();
|
||||
let (versions, next) = parse_collection_page(json).unwrap();
|
||||
assert_eq!(versions, vec!["5.1.0-1ubuntu2", "2:1.0-1"]);
|
||||
// A page without a next link is the end of the collection
|
||||
assert_eq!(next, None);
|
||||
}
|
||||
|
||||
/// A 200 answer with zero entries is the normal "nothing published
|
||||
/// there" case, and must deserialize to an empty list
|
||||
#[test]
|
||||
fn published_sources_parses_empty_collection() {
|
||||
let sources: PublishedSources =
|
||||
serde_json::from_str(r#"{"start": 0, "total_size": 0, "entries": []}"#).unwrap();
|
||||
assert!(sources.entries.is_empty());
|
||||
let (versions, next) =
|
||||
parse_collection_page(r#"{"start": 0, "total_size": 0, "entries": []}"#).unwrap();
|
||||
assert!(versions.is_empty());
|
||||
assert_eq!(next, None);
|
||||
}
|
||||
|
||||
/// A truncated page announces the next one through
|
||||
/// `next_collection_link`, carried as a plain JSON string (shape
|
||||
/// verified against the live API)
|
||||
#[test]
|
||||
fn parse_collection_page_reads_next_link() {
|
||||
let json = r#"{
|
||||
"start": 0,
|
||||
"total_size": 150,
|
||||
"entries": [{"source_package_version": "1.0-1"}],
|
||||
"next_collection_link": "https://api.launchpad.net/1.0/~u/+archive/ubuntu/p?ws.op=getPublishedSources&ws.size=100&memo=100&ws.start=100"
|
||||
}"#;
|
||||
|
||||
let (versions, next) = parse_collection_page(json).unwrap();
|
||||
assert_eq!(versions, vec!["1.0-1"]);
|
||||
assert_eq!(
|
||||
next.as_deref(),
|
||||
Some(
|
||||
"https://api.launchpad.net/1.0/~u/+archive/ubuntu/p?ws.op=getPublishedSources&ws.size=100&memo=100&ws.start=100"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/// Serve canned byte responses on a local port, one per connection (the
|
||||
/// last response repeats), and return the listener for URL building
|
||||
///
|
||||
/// The canned responses must use 'Connection: close' so the client opens
|
||||
/// a fresh connection (and receives a fresh response) per request.
|
||||
fn serve_responses(listener: std::net::TcpListener, responses: Vec<String>) {
|
||||
use std::io::{Read, Write};
|
||||
std::thread::spawn(move || {
|
||||
for (served, mut stream) in listener.incoming().flatten().enumerate() {
|
||||
let index = served.min(responses.len() - 1);
|
||||
// Drain the request first: closing with unread inbound data
|
||||
// would send a TCP RST and destroy the response in flight
|
||||
let mut buf = [0u8; 4096];
|
||||
loop {
|
||||
match stream.read(&mut buf) {
|
||||
Ok(0) => break,
|
||||
Ok(n) if buf[..n].windows(4).any(|w| w == b"\r\n\r\n") => break,
|
||||
Ok(_) => continue,
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
let body = &responses[index];
|
||||
let _ = stream.write_all(
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
let _ = stream.flush();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// One `getPublishedSources` page carrying `versions`, with the
|
||||
/// `next_collection_link` of a truncated page when `next` is given
|
||||
fn collection_body(versions: &[&str], next: Option<&str>) -> String {
|
||||
let entries: Vec<String> = versions
|
||||
.iter()
|
||||
.map(|v| format!(r#"{{"source_package_version": "{v}"}}"#))
|
||||
.collect();
|
||||
let next_field = next
|
||||
.map(|link| format!(r#", "next_collection_link": "{link}""#))
|
||||
.unwrap_or_default();
|
||||
format!(
|
||||
r#"{{"start": 0, "total_size": {}, "entries": [{}]{next_field}}}"#,
|
||||
versions.len(),
|
||||
entries.join(", ")
|
||||
)
|
||||
}
|
||||
|
||||
/// Bind a fresh mock server ready to serve `responses` (the caller
|
||||
/// needs the address to build self-referential `next_collection_link`s
|
||||
/// before serving starts)
|
||||
fn bound_collection_server() -> (std::net::TcpListener, String) {
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||
(listener, base)
|
||||
}
|
||||
|
||||
/// The collection walk follows `next_collection_link`: the versions of
|
||||
/// every page are collected in order, and the walk stops on the page
|
||||
/// without a next link (the mock repeats its last response forever, so
|
||||
/// an extra fetch would still pass — but a missing next-link handling
|
||||
/// would drop page two's versions from the result)
|
||||
#[tokio::test]
|
||||
async fn walk_collection_collects_every_page() {
|
||||
let (listener, base) = bound_collection_server();
|
||||
serve_responses(
|
||||
listener,
|
||||
vec![
|
||||
collection_body(&["1.0-1", "1.6-1"], Some(&format!("{base}/next"))),
|
||||
collection_body(&["0.9-1"], None),
|
||||
],
|
||||
);
|
||||
|
||||
let versions = walk_collection(
|
||||
crate::distro_info::http_client(),
|
||||
&format!("{base}/~u/+archive/ubuntu/p?ws.op=getPublishedSources"),
|
||||
"u/p",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(versions, vec!["1.0-1", "1.6-1", "0.9-1"]);
|
||||
}
|
||||
|
||||
/// A next-link chain that never ends must error, not loop forever: the
|
||||
/// partial list would feed the superseded check a false "not superseded"
|
||||
#[tokio::test]
|
||||
async fn walk_collection_errors_when_pagination_never_ends() {
|
||||
// The last (only) response repeats forever, each page linking back
|
||||
// to the server: the walk must stop at the page cap by itself
|
||||
let (listener, base) = bound_collection_server();
|
||||
serve_responses(
|
||||
listener,
|
||||
vec![collection_body(&["1.0-1"], Some(&format!("{base}/loop")))],
|
||||
);
|
||||
|
||||
let err = walk_collection(
|
||||
crate::distro_info::http_client(),
|
||||
&format!("{base}/~u/+archive/ubuntu/p?ws.op=getPublishedSources"),
|
||||
"u/p",
|
||||
)
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(
|
||||
err.contains("keeps paginating the published sources of 'u/p'"),
|
||||
"unexpected: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -282,11 +282,9 @@ pub fn read_pyproject(path: &Path) -> Option<PyProject> {
|
||||
("project", "description") => project.description = Some(value),
|
||||
("project", "license") => project.license = license_text(&value),
|
||||
("project.urls", "Homepage") => project.homepage = Some(value),
|
||||
("project.scripts", key) => {
|
||||
if project.script.is_none() {
|
||||
("project.scripts", key) if project.script.is_none() => {
|
||||
project.script = Some(key.to_string());
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
+359
-16
@@ -6,7 +6,10 @@
|
||||
//!
|
||||
//! Payload files are uploaded first and the `.changes` file last, like
|
||||
//! dput does, so a partially uploaded set cannot be picked up by the
|
||||
//! server-side queue processors.
|
||||
//! server-side queue processors. A run that fails mid-upload removes its
|
||||
//! already-uploaded files from the incoming queue (best effort), so a
|
||||
//! retried upload starts from a clean queue; a failed upload is never
|
||||
//! recorded in the upload log, so a re-run replays every file.
|
||||
|
||||
pub mod changes;
|
||||
pub mod ssh;
|
||||
@@ -96,7 +99,15 @@ pub async fn put(
|
||||
// Pre-flight checks for everything the upload queue only rejects after
|
||||
// processing: a valid Section, a known target series, and the target
|
||||
// PPA actually existing (the SFTP queue itself is a blind write)
|
||||
check_control_section(&opts.cwd, "ubuntu")?;
|
||||
match section_check_target(opts.changes.as_deref(), &opts.cwd) {
|
||||
SectionCheckTarget::Dir(dir) => check_control_section(&dir, "ubuntu")?,
|
||||
SectionCheckTarget::Skip => log::warn!(
|
||||
"cannot check the Section: '{}' has no debian/control next to \
|
||||
it, skipping the pre-flight (the archive still rejects uploads \
|
||||
with unknown sections)",
|
||||
changes_path.display()
|
||||
),
|
||||
}
|
||||
|
||||
let checking = multi.add(ProgressBar::new(0));
|
||||
checking.set_style(ui::spinner_style());
|
||||
@@ -147,23 +158,50 @@ pub async fn put(
|
||||
.to_string();
|
||||
uploads.push((changes_path.clone(), changes_name.clone()));
|
||||
|
||||
let incoming = target.incoming.trim_end_matches('/');
|
||||
|
||||
// Remote names of the files uploaded so far, in upload order (the
|
||||
// .changes last). A run failing mid-upload removes these from the
|
||||
// write-only incoming queue before returning: the uploaded payloads
|
||||
// would otherwise linger in the queue area forever, and a .changes
|
||||
// truncated by a failed close could even be picked up by the scanner.
|
||||
let mut uploaded: Vec<String> = Vec::new();
|
||||
|
||||
for (path, name) in &uploads {
|
||||
let size = path
|
||||
.metadata()
|
||||
.map_err(|e| format!("cannot stat '{}': {}", path.display(), e))?
|
||||
.len();
|
||||
let size = match path.metadata() {
|
||||
Ok(metadata) => metadata.len(),
|
||||
Err(e) => {
|
||||
// Nothing was attempted for this file: only what earlier
|
||||
// iterations uploaded needs removing
|
||||
let error: Box<dyn std::error::Error> =
|
||||
format!("cannot stat '{}': {}", path.display(), e).into();
|
||||
cleanup_partial_upload(&sftp, incoming, &uploaded, None, &host);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
// Same transfer view as pull: prefix line, bar on its own line
|
||||
let bar = multi.add(ProgressBar::new(size));
|
||||
bar.enable_steady_tick(std::time::Duration::from_millis(50));
|
||||
bar.set_style(ui::transfer_style());
|
||||
bar.set_prefix(format!("Uploading {name}..."));
|
||||
|
||||
let remote = format!("{}/{}", target.incoming.trim_end_matches('/'), name);
|
||||
let result = ssh::upload_file(&sftp, path, &remote, &bar);
|
||||
let remote = format!("{incoming}/{name}");
|
||||
let result = ssh::upload_file(&sftp, path, &remote, &host, &bar);
|
||||
bar.finish_and_clear();
|
||||
result?;
|
||||
if let Err(e) = result {
|
||||
// The failed file itself joins the cleanup: its remote `create`
|
||||
// may have succeeded before the failure, leaving a partial — or,
|
||||
// on a failed close, a truncated — file behind
|
||||
cleanup_partial_upload(&sftp, incoming, &uploaded, Some(name), &host);
|
||||
return Err(e);
|
||||
}
|
||||
uploaded.push(name.clone());
|
||||
}
|
||||
|
||||
// Recorded only once the whole upload succeeded: the log backs the
|
||||
// duplicate-upload guard, and a failed upload must not count as
|
||||
// uploaded (a re-run replays every file — `sftp.create` truncates, so
|
||||
// replaying is safe).
|
||||
record_upload(&upload_log_path()?, &record)?;
|
||||
|
||||
// The completion lines replace the summary bar; the guard's own clear
|
||||
@@ -180,6 +218,48 @@ pub async fn put(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The remote names to attempt removing after a failed upload: everything
|
||||
/// already uploaded plus, when set, `failed` (the file whose upload just
|
||||
/// failed — its remote `create` may have succeeded before the failure,
|
||||
/// leaving a partial or truncated file behind) — in reverse upload order,
|
||||
/// so a `.changes` is removed before the payloads it references and the
|
||||
/// queue scanner never observes the payload set shrinking under a
|
||||
/// still-present `.changes`. Pure so the ordering decision is testable
|
||||
/// without a server; the network side is [`cleanup_partial_upload`].
|
||||
fn cleanup_list(uploaded: &[String], failed: Option<&str>) -> Vec<String> {
|
||||
let mut names: Vec<String> = uploaded.to_vec();
|
||||
if let Some(failed) = failed {
|
||||
names.push(failed.to_string());
|
||||
}
|
||||
names.reverse();
|
||||
names
|
||||
}
|
||||
|
||||
/// Best-effort removal of what a failed upload left in the target's
|
||||
/// incoming queue: the already-uploaded payloads, and a `.changes`
|
||||
/// truncated by a failed close, would otherwise linger in the write-only
|
||||
/// area until the queue is manually cleaned. Launchpad re-validates every
|
||||
/// upload, so this is pure hygiene: a removal failure is logged and
|
||||
/// skipped, and the caller returns the original upload error — never a
|
||||
/// cleanup one.
|
||||
fn cleanup_partial_upload(
|
||||
sftp: &ssh2::Sftp,
|
||||
incoming: &str,
|
||||
uploaded: &[String],
|
||||
failed: Option<&str>,
|
||||
host: &str,
|
||||
) {
|
||||
for name in cleanup_list(uploaded, failed) {
|
||||
let remote = format!("{incoming}/{name}");
|
||||
match ssh::remove_file(sftp, &remote, host) {
|
||||
Ok(()) => info!("Removed leftover {remote} from the failed upload"),
|
||||
Err(e) => {
|
||||
log::warn!("Could not remove the leftover {remote} of the failed upload: {e}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Steady tick interval of every bar rendered by a `put` run
|
||||
const TICK: std::time::Duration = std::time::Duration::from_millis(50);
|
||||
|
||||
@@ -251,6 +331,40 @@ fn check_control_section(cwd: &Path, dist: &str) -> Result<(), Box<dyn std::erro
|
||||
.into())
|
||||
}
|
||||
|
||||
/// What the Section pre-flight decided to validate: a directory holding a
|
||||
/// `debian/control` ([`check_control_section`]'s expected layout), or that
|
||||
/// it cannot run where the upload lives and must be skipped.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
enum SectionCheckTarget {
|
||||
/// Validate `debian/control` in this directory
|
||||
Dir(PathBuf),
|
||||
/// No `debian/control` next to the `.changes`: skip the check
|
||||
Skip,
|
||||
}
|
||||
|
||||
/// Decide which tree the Section pre-flight validates. An explicit
|
||||
/// `--changes` file must be followed by its own directory — it can name a
|
||||
/// different package than the current tree — so the check runs against the
|
||||
/// `.changes`' directory when it directly holds a `debian/control` (the
|
||||
/// `.changes` inside the package root), and is skipped otherwise: pkh
|
||||
/// writes the `.changes` next to the source tree (an artifacts-only
|
||||
/// directory as far as `debian/control` goes), and checking `cwd` there
|
||||
/// would validate whatever package the user happens to be in. Without
|
||||
/// `--changes` the upload comes from the current tree, which keeps getting
|
||||
/// checked exactly as before. Pure decision, factored out so the
|
||||
/// wrong-tree rule is testable with plain tempdirs.
|
||||
fn section_check_target(changes_path: Option<&Path>, cwd: &Path) -> SectionCheckTarget {
|
||||
let Some(changes) = changes_path else {
|
||||
return SectionCheckTarget::Dir(cwd.to_path_buf());
|
||||
};
|
||||
let dir = changes.parent().unwrap_or_else(|| Path::new("."));
|
||||
if dir.join("debian/control").is_file() {
|
||||
SectionCheckTarget::Dir(dir.to_path_buf())
|
||||
} else {
|
||||
SectionCheckTarget::Skip
|
||||
}
|
||||
}
|
||||
|
||||
/// The highest version among `published` that parses as a Debian version,
|
||||
/// `None` when the list is empty or nothing in it parses (entries that
|
||||
/// cannot be parsed are skipped rather than failing the check: the API
|
||||
@@ -419,7 +533,7 @@ fn discover_changes(cwd: &Path) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
||||
/// upload date. Matching on the digest makes `--force` the only way to
|
||||
/// re-upload an identical file, while a rebuilt file (new digest) never
|
||||
/// trips the guard.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
struct UploadRecord {
|
||||
target: String,
|
||||
file: String,
|
||||
@@ -461,12 +575,52 @@ fn upload_log_path() -> Result<PathBuf, Box<dyn std::error::Error>> {
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
/// Load the upload log, empty when the file does not exist yet
|
||||
/// Load the upload log, empty when the file does not exist yet.
|
||||
///
|
||||
/// The log only backs the advisory duplicate-upload guard (Launchpad
|
||||
/// re-verifies every upload), so a corrupt log must not abort the upload:
|
||||
/// like the context configuration in [`crate::context::manager`], the corrupt
|
||||
/// file is first backed up to `<path>.bak` (best effort) so a later
|
||||
/// [`record_upload`] cannot destroy it, then an empty log is used.
|
||||
fn load_upload_log(path: &Path) -> Vec<UploadRecord> {
|
||||
std::fs::read_to_string(path)
|
||||
.ok()
|
||||
.and_then(|content| serde_json::from_str(&content).ok())
|
||||
.unwrap_or_default()
|
||||
let content = match std::fs::read_to_string(path) {
|
||||
Ok(content) => content,
|
||||
Err(e) => {
|
||||
if e.kind() != std::io::ErrorKind::NotFound {
|
||||
log::error!(
|
||||
"Cannot read upload log {}: {e}; continuing as if nothing had been uploaded",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
match serde_json::from_str(&content) {
|
||||
Ok(entries) => entries,
|
||||
Err(e) => {
|
||||
log::error!(
|
||||
"Upload log {} is corrupt ({e}); ignoring its content, earlier uploads may not be detected as duplicates anymore",
|
||||
path.display()
|
||||
);
|
||||
backup_corrupt_log(path);
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Back up a corrupt upload log (best effort) so a later
|
||||
/// [`record_upload`] cannot silently destroy its content.
|
||||
fn backup_corrupt_log(path: &Path) {
|
||||
let mut os = path.as_os_str().to_os_string();
|
||||
os.push(".bak");
|
||||
let backup_path = PathBuf::from(os);
|
||||
match std::fs::copy(path, &backup_path) {
|
||||
Ok(_) => log::warn!("Corrupt upload log backed up to {}", backup_path.display()),
|
||||
Err(e) => log::warn!(
|
||||
"Could not back up corrupt upload log to {}: {e}",
|
||||
backup_path.display()
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// The previous upload of `record` from the log at `log`, if any (same
|
||||
@@ -484,7 +638,16 @@ fn find_previous_upload(
|
||||
fn record_upload(log: &Path, record: &UploadRecord) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut entries = load_upload_log(log);
|
||||
entries.push(record.clone());
|
||||
std::fs::write(log, serde_json::to_string_pretty(&entries)?)?;
|
||||
// Write to a temporary file next to the log, then rename it over the
|
||||
// log: rename is atomic within a filesystem, so a crash mid-write leaves
|
||||
// the previous log intact instead of a truncated (corrupt) file.
|
||||
// The temporary file is created with default permissions; the log holds
|
||||
// no secrets and lives in the user's data directory, so the previous
|
||||
// mode is not carried over (same choice as `Files::save_atomic`).
|
||||
let tmp = log.with_extension("new");
|
||||
std::fs::write(&tmp, serde_json::to_string_pretty(&entries)?)
|
||||
.map_err(|e| format!("cannot write '{}': {}", tmp.display(), e))?;
|
||||
std::fs::rename(&tmp, log).map_err(|e| format!("cannot install '{}': {}", log.display(), e))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -578,6 +741,75 @@ mod tests {
|
||||
assert!(err.contains("has no Section"), "unexpected: {err}");
|
||||
}
|
||||
|
||||
/// An explicit `--changes` file of a different package must be checked
|
||||
/// in its own directory (A), not in the current tree (B): B's control
|
||||
/// file decided the pre-flight before the fix
|
||||
#[test]
|
||||
fn section_check_follows_explicit_changes_directory() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let artifacts = dir.path().join("artifacts");
|
||||
let elsewhere = dir.path().join("elsewhere");
|
||||
std::fs::create_dir_all(artifacts.join("debian")).unwrap();
|
||||
std::fs::create_dir_all(elsewhere.join("debian")).unwrap();
|
||||
let changes = artifacts.join("hello_1.0-1_source.changes");
|
||||
std::fs::write(&changes, b"changes").unwrap();
|
||||
std::fs::write(
|
||||
artifacts.join("debian/control"),
|
||||
"Source: hello\nSection: utils\n",
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
elsewhere.join("debian/control"),
|
||||
"Source: other\nSection: unknown\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
section_check_target(Some(&changes), &elsewhere),
|
||||
SectionCheckTarget::Dir(artifacts.clone())
|
||||
);
|
||||
|
||||
// The decided directory carries the valid section, while the tree
|
||||
// the old code checked would have failed the upload
|
||||
match section_check_target(Some(&changes), &elsewhere) {
|
||||
SectionCheckTarget::Dir(d) => check_control_section(&d, "ubuntu").unwrap(),
|
||||
SectionCheckTarget::Skip => panic!("the .changes directory has debian/control"),
|
||||
}
|
||||
assert!(check_control_section(&elsewhere, "ubuntu").is_err());
|
||||
}
|
||||
|
||||
/// A `.changes` in a directory without `debian/control` — pkh's own
|
||||
/// layout puts it next to the source tree, and artifacts can be
|
||||
/// collected away from any source — skips the check instead of
|
||||
/// validating whatever tree `cwd` points at
|
||||
#[test]
|
||||
fn section_check_skips_when_changes_dir_has_no_control() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let artifacts = dir.path().join("artifacts");
|
||||
std::fs::create_dir_all(&artifacts).unwrap();
|
||||
let changes = artifacts.join("hello_1.0-1_source.changes");
|
||||
std::fs::write(&changes, b"changes").unwrap();
|
||||
|
||||
assert_eq!(
|
||||
section_check_target(Some(&changes), dir.path()),
|
||||
SectionCheckTarget::Skip
|
||||
);
|
||||
}
|
||||
|
||||
/// Without `--changes` the upload comes from the current tree: the
|
||||
/// decision stays the cwd, byte-identical to the pre-fix behavior
|
||||
#[test]
|
||||
fn section_check_without_changes_validates_cwd() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let pkg = dir.path().join("hello");
|
||||
std::fs::create_dir_all(pkg.join("debian")).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
section_check_target(None, &pkg),
|
||||
SectionCheckTarget::Dir(pkg)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn discover_errors_when_several_candidates() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
@@ -632,6 +864,65 @@ mod tests {
|
||||
assert!(find_previous_upload(&log, &rebuilt).unwrap().is_none());
|
||||
}
|
||||
|
||||
/// A plain upload record for the log tests
|
||||
fn upload_log_record(sha256: &str) -> UploadRecord {
|
||||
UploadRecord {
|
||||
target: "ppa:user/ppa".to_string(),
|
||||
file: "hello_1.0-1_source.changes".to_string(),
|
||||
sha256: sha256.to_string(),
|
||||
date: "2025-09-16 12:00".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A corrupt log must not be silently reset: it is backed up byte for
|
||||
/// byte, and the guard restarts from an empty log.
|
||||
#[test]
|
||||
fn corrupt_upload_log_is_backed_up_and_reset() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let log = dir.path().join("uploads.json");
|
||||
let corrupt = b"{ \"not\": valid json";
|
||||
std::fs::write(&log, corrupt).unwrap();
|
||||
|
||||
assert!(load_upload_log(&log).is_empty());
|
||||
|
||||
let backup = dir.path().join("uploads.json.bak");
|
||||
assert_eq!(std::fs::read(&backup).unwrap(), corrupt);
|
||||
|
||||
// Recording after a corrupt log starts a fresh, parseable log
|
||||
// without clobbering the backup
|
||||
record_upload(&log, &upload_log_record("abc")).unwrap();
|
||||
assert_eq!(load_upload_log(&log), vec![upload_log_record("abc")]);
|
||||
assert_eq!(std::fs::read(&backup).unwrap(), corrupt);
|
||||
}
|
||||
|
||||
/// `record_upload` leaves a log that parses back, with no temporary
|
||||
/// file left behind
|
||||
#[test]
|
||||
fn recorded_upload_parses_back_atomically() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let log = dir.path().join("uploads.json");
|
||||
|
||||
record_upload(&log, &upload_log_record("abc")).unwrap();
|
||||
|
||||
assert_eq!(load_upload_log(&log), vec![upload_log_record("abc")]);
|
||||
assert!(!dir.path().join("uploads.new").exists());
|
||||
}
|
||||
|
||||
/// Recording a second upload appends to the log instead of replacing it
|
||||
#[test]
|
||||
fn record_over_existing_log_preserves_prior_entries() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let log = dir.path().join("uploads.json");
|
||||
|
||||
record_upload(&log, &upload_log_record("abc")).unwrap();
|
||||
record_upload(&log, &upload_log_record("def")).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
load_upload_log(&log),
|
||||
vec![upload_log_record("abc"), upload_log_record("def")]
|
||||
);
|
||||
}
|
||||
|
||||
/// A minimal in-memory changes file for the superseded-version check
|
||||
fn changes_fixture(version: &str) -> changes::ChangesFile {
|
||||
changes::ChangesFile {
|
||||
@@ -756,4 +1047,56 @@ mod tests {
|
||||
let changes = changes_fixture("1.7-1");
|
||||
check_not_superseded(&target, &changes, &published, false).unwrap();
|
||||
}
|
||||
|
||||
/// The upload loop is network-bound and `ssh2::Sftp` cannot be built
|
||||
/// without a live SSH session, so there is no stub seam for it without
|
||||
/// heavy refactoring; the cleanup decision is factored into the pure
|
||||
/// [`cleanup_list`] and tested directly instead.
|
||||
///
|
||||
/// Failure during the final `.changes` upload (the dangerous case: a
|
||||
/// partial or close-truncated `.changes` is scanner-visible): the
|
||||
/// `.changes` is removed first, then the payloads it references, so the
|
||||
/// queue never sees a `.changes` over a shrinking payload set.
|
||||
#[test]
|
||||
fn cleanup_list_removes_changes_first_then_payloads_in_reverse() {
|
||||
let uploaded = vec![
|
||||
"hello_1.0.orig.tar.xz".to_string(),
|
||||
"hello_1.0-1.debian.tar.xz".to_string(),
|
||||
"hello_1.0-1.dsc".to_string(),
|
||||
];
|
||||
assert_eq!(
|
||||
cleanup_list(&uploaded, Some("hello_1.0-1_source.changes")),
|
||||
vec![
|
||||
"hello_1.0-1_source.changes",
|
||||
"hello_1.0-1.dsc",
|
||||
"hello_1.0-1.debian.tar.xz",
|
||||
"hello_1.0.orig.tar.xz",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// A failure during a payload upload: the `.changes` was never
|
||||
/// attempted (it is uploaded last), so it must not appear in the
|
||||
/// cleanup list — only the failed payload (a partial may exist) and the
|
||||
/// earlier payloads.
|
||||
#[test]
|
||||
fn cleanup_list_before_the_changes_covers_the_failed_payload() {
|
||||
let uploaded = vec!["hello_1.0.orig.tar.xz".to_string()];
|
||||
assert_eq!(
|
||||
cleanup_list(&uploaded, Some("hello_1.0-1.debian.tar.xz")),
|
||||
vec!["hello_1.0-1.debian.tar.xz", "hello_1.0.orig.tar.xz"]
|
||||
);
|
||||
}
|
||||
|
||||
/// A stat failure happens before anything was attempted for that file,
|
||||
/// so only what earlier iterations uploaded is removed — and with
|
||||
/// nothing uploaded yet there is nothing to remove at all.
|
||||
#[test]
|
||||
fn cleanup_list_without_failed_file_keeps_uploaded_only() {
|
||||
assert!(cleanup_list(&[], None).is_empty());
|
||||
assert_eq!(
|
||||
cleanup_list(&["hello_1.0.orig.tar.xz".to_string()], None),
|
||||
vec!["hello_1.0.orig.tar.xz"]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+378
-24
@@ -5,21 +5,27 @@
|
||||
//! authentication (every ssh-agent identity first, then configured and
|
||||
//! default key files) and chunked SFTP upload with progress reporting.
|
||||
//!
|
||||
//! Every network phase is time-bounded — the TCP connect, the SSH
|
||||
//! handshake/authentication, and each low-level call of the SFTP data
|
||||
//! transfer (see the `*_TIMEOUT` constants) — so a black-holed or stalled
|
||||
//! server fails the upload instead of hanging it forever.
|
||||
//!
|
||||
//! This replaces dput-ng's paramiko transport with the `ssh2` (libssh2)
|
||||
//! stack the rest of pkh already uses for build contexts.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpStream;
|
||||
use std::net::{SocketAddr, TcpStream, ToSocketAddrs};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use indicatif::ProgressBar;
|
||||
use lazy_static::lazy_static;
|
||||
use log::debug;
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
use ssh2::{CheckResult, HostKeyType, KnownHostFileKind, Session};
|
||||
use ssh2::{CheckResult, HostKeyType, KnownHostFileKind, KnownHosts, Session};
|
||||
|
||||
use crate::ui::prompt;
|
||||
|
||||
@@ -221,19 +227,103 @@ fn wildmatch(text: &str, pattern: &str) -> bool {
|
||||
pattern[p..].iter().all(|&c| c == '*')
|
||||
}
|
||||
|
||||
/// Bound on one TCP connection attempt to one resolved address: `connect(2)`
|
||||
/// would otherwise block for minutes (or forever, behind a silent firewall)
|
||||
/// per address. Generous enough for slow links to Launchpad, short enough
|
||||
/// that a dead target fails in seconds.
|
||||
const TCP_CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
|
||||
/// Timeout for the blocking libssh2 calls of the connection phase (banner
|
||||
/// exchange and key exchange of the handshake, host key check,
|
||||
/// authentication): when it expires the pending call fails with
|
||||
/// `LIBSSH2_ERROR_TIMEOUT` (surfaced as "timed out") instead of hanging
|
||||
/// forever on a stalled server. See [`connect`] for what this does and does
|
||||
/// not cover.
|
||||
const SSH_API_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Timeout for the blocking libssh2 calls of the SFTP data transfer (file
|
||||
/// creation, chunk writes, final close handshake). It applies *per
|
||||
/// low-level libssh2 call*, not to the whole transfer: the clock restarts
|
||||
/// at every API entry, so the wall-clock duration of a transfer is not
|
||||
/// bounded by design — but a stalled or black-holed server fails one call
|
||||
/// after this budget instead of hanging `pkh put` forever.
|
||||
const SSH_TRANSFER_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
|
||||
/// libssh2 timeouts are millisecond counts; the constants above are exact
|
||||
/// multiples of a millisecond, and anything larger saturates instead of
|
||||
/// truncating
|
||||
fn duration_ms(timeout: Duration) -> u32 {
|
||||
u32::try_from(timeout.as_millis()).unwrap_or(u32::MAX)
|
||||
}
|
||||
|
||||
/// Resolve `host:port` and open a TCP connection, trying each resolved
|
||||
/// address in order (like `TcpStream::connect` does) with
|
||||
/// [`TCP_CONNECT_TIMEOUT`] per attempt instead of blocking indefinitely.
|
||||
/// Fails with a message naming the target and every per-address error.
|
||||
fn tcp_connect(host: &str, port: u16) -> Result<TcpStream, String> {
|
||||
let addrs: Vec<SocketAddr> = (host, port)
|
||||
.to_socket_addrs()
|
||||
.map_err(|e| format!("cannot resolve {host}:{port}: {e}"))?
|
||||
.collect();
|
||||
if addrs.is_empty() {
|
||||
return Err(format!(
|
||||
"cannot connect to {host}:{port}: '{host}' resolved to no addresses"
|
||||
));
|
||||
}
|
||||
|
||||
let mut attempts: Vec<(SocketAddr, std::io::Error)> = Vec::new();
|
||||
for addr in addrs {
|
||||
match TcpStream::connect_timeout(&addr, TCP_CONNECT_TIMEOUT) {
|
||||
Ok(stream) => return Ok(stream),
|
||||
Err(e) => attempts.push((addr, e)),
|
||||
}
|
||||
}
|
||||
Err(connect_failed_message(host, port, &attempts))
|
||||
}
|
||||
|
||||
/// Message for a target none of whose resolved addresses accepted a
|
||||
/// connection within [`TCP_CONNECT_TIMEOUT`]. Pure so tests can assert it
|
||||
/// without any network.
|
||||
fn connect_failed_message(
|
||||
host: &str,
|
||||
port: u16,
|
||||
attempts: &[(SocketAddr, std::io::Error)],
|
||||
) -> String {
|
||||
let details = attempts
|
||||
.iter()
|
||||
.map(|(addr, e)| format!("{addr}: {e}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("; ");
|
||||
format!(
|
||||
"cannot connect to {host}:{port} (tried {} address(es)): {details}",
|
||||
attempts.len()
|
||||
)
|
||||
}
|
||||
|
||||
/// Connect to `host:port`, verify the server host key and authenticate as
|
||||
/// `login`: every ssh-agent identity first, then the configured and default
|
||||
/// identity files.
|
||||
/// identity files. The TCP connect, handshake and authentication steps are
|
||||
/// time-bounded ([`TCP_CONNECT_TIMEOUT`], [`SSH_API_TIMEOUT`]).
|
||||
pub fn connect(
|
||||
host: &str,
|
||||
port: u16,
|
||||
login: &str,
|
||||
config: &SshConfig,
|
||||
) -> Result<Session, Box<dyn std::error::Error>> {
|
||||
let tcp = TcpStream::connect((host, port))
|
||||
.map_err(|e| format!("cannot connect to {host}:{port}: {e}"))?;
|
||||
let tcp = tcp_connect(host, port)?;
|
||||
|
||||
let mut session = Session::new()?;
|
||||
// In blocking mode (the libssh2 default), a call that would block loops
|
||||
// in `_libssh2_wait_socket` (via the `BLOCK_ADJUST` macros of
|
||||
// session.h in the vendored libssh2-sys sources), which bounds the
|
||||
// underlying poll()/select() wait by this timeout and fails the call
|
||||
// with LIBSSH2_ERROR_TIMEOUT when it expires. Verified against
|
||||
// libssh2-sys 0.3.3: the handshake (session.c), host key check,
|
||||
// authentication, channel and SFTP calls all route through it, so a
|
||||
// stalled server errors out instead of hanging forever. It bounds
|
||||
// *each* libssh2 call, not the whole phase: a server that trickles
|
||||
// bytes often enough keeps resuming every call in time.
|
||||
session.set_timeout(duration_ms(SSH_API_TIMEOUT));
|
||||
session.set_tcp_stream(tcp);
|
||||
session
|
||||
.handshake()
|
||||
@@ -246,13 +336,19 @@ pub fn connect(
|
||||
|
||||
authenticate(&session, host, login, config)?;
|
||||
|
||||
// Only SFTP open/data calls remain on this session: switch from the
|
||||
// connection-phase budget to the generous per-call transfer one
|
||||
session.set_timeout(duration_ms(SSH_TRANSFER_TIMEOUT));
|
||||
|
||||
Ok(session)
|
||||
}
|
||||
|
||||
/// Check the server's host key, in decreasing order of trust:
|
||||
///
|
||||
/// 1. matching one of the host's pinned (published, `host_keys.yml`)
|
||||
/// fingerprints — accepted silently;
|
||||
/// fingerprints — accepted (with a diagnostic warning when the user's
|
||||
/// known_hosts records a *different* key for the host, so a stale
|
||||
/// entry gets cleaned up; the pin stays authoritative);
|
||||
/// 2. matching `~/.ssh/known_hosts` — accepted;
|
||||
/// 3. known-and-different — refused loudly (possible man-in-the-middle);
|
||||
/// 4. unknown — fingerprint shown, explicit confirmation required, and on
|
||||
@@ -268,21 +364,16 @@ fn verify_host_key(
|
||||
|
||||
if host_key_is_pinned(host, &fingerprint) {
|
||||
debug!("{host} host key matches the published fingerprint");
|
||||
// Purely diagnostic — the pin already decided — but a known_hosts
|
||||
// entry disagreeing with the published key is worth surfacing
|
||||
log_known_hosts_discrepancy(host, port, key);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut known_hosts = Session::new()?.known_hosts()?;
|
||||
for file in known_hosts_files() {
|
||||
// Unreadable/unknown-format lines are skipped by libssh2; a missing
|
||||
// file is fine
|
||||
let _ = known_hosts.read_file(&file, KnownHostFileKind::OpenSSH);
|
||||
}
|
||||
load_known_hosts(&mut known_hosts, &known_hosts_files());
|
||||
|
||||
let check = if port == 22 {
|
||||
known_hosts.check(host, key)
|
||||
} else {
|
||||
known_hosts.check_port(host, port, key)
|
||||
};
|
||||
let check = check_known_hosts(&known_hosts, host, port, key);
|
||||
|
||||
match check {
|
||||
CheckResult::Match => Ok(()),
|
||||
@@ -320,6 +411,76 @@ fn verify_host_key(
|
||||
}
|
||||
}
|
||||
|
||||
/// Read `files` into `known_hosts`: silently skips missing files (the
|
||||
/// normal first-use case — there is nothing to consult), warns about a
|
||||
/// file that exists but cannot be read or parsed, naming the file and the
|
||||
/// error — the user should know why their maintained entries are not being
|
||||
/// consulted — and skips it. Unreadable/unknown-format *lines* are skipped
|
||||
/// by libssh2 itself. Returns the emitted warnings, so the decision is
|
||||
/// testable without capturing log output.
|
||||
fn load_known_hosts(known_hosts: &mut KnownHosts, files: &[PathBuf]) -> Vec<String> {
|
||||
let mut warnings = Vec::new();
|
||||
for file in files {
|
||||
if !file.exists() {
|
||||
continue;
|
||||
}
|
||||
if let Err(e) = known_hosts.read_file(file, KnownHostFileKind::OpenSSH) {
|
||||
let warning = format!(
|
||||
"cannot read the known hosts file {}: {e} — its entries are ignored",
|
||||
file.display()
|
||||
);
|
||||
log::warn!("{warning}");
|
||||
warnings.push(warning);
|
||||
}
|
||||
}
|
||||
warnings
|
||||
}
|
||||
|
||||
/// Look up `host`'s `key` in the loaded entries, using the `[host]:port`
|
||||
/// spelling for non-standard ports (the known_hosts encoding)
|
||||
fn check_known_hosts(known_hosts: &KnownHosts, host: &str, port: u16, key: &[u8]) -> CheckResult {
|
||||
if port == 22 {
|
||||
known_hosts.check(host, key)
|
||||
} else {
|
||||
known_hosts.check_port(host, port, key)
|
||||
}
|
||||
}
|
||||
|
||||
/// What to tell the user when the server's key matched a pinned (published)
|
||||
/// fingerprint while their known_hosts records a different key for the same
|
||||
/// host — a stale entry, or worse an old compromise artifact they should
|
||||
/// clean up; `None` for every other check result, since the pin is
|
||||
/// authoritative and needs no corroboration. Pure so the decision and the
|
||||
/// wording are testable without a server or a filesystem.
|
||||
fn known_hosts_discrepancy(host: &str, check: CheckResult) -> Option<String> {
|
||||
match check {
|
||||
CheckResult::Mismatch => Some(format!(
|
||||
"the host key of {host} matches Launchpad's published \
|
||||
fingerprint, but your known_hosts file(s) record a DIFFERENT \
|
||||
key for it — proceeding on the published fingerprint; consider \
|
||||
removing the stale '{host}' entry from your known_hosts"
|
||||
)),
|
||||
CheckResult::Match | CheckResult::NotFound | CheckResult::Failure => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Diagnostic-only companion to the pinned-fingerprint acceptance: load the
|
||||
/// user's known hosts files and warn when they record a different key for
|
||||
/// `host` than the pinned one about to be accepted. The pin stays
|
||||
/// authoritative: any failure here is tolerated and the connection
|
||||
/// proceeds regardless.
|
||||
fn log_known_hosts_discrepancy(host: &str, port: u16, key: &[u8]) {
|
||||
let Ok(mut known_hosts) = Session::new().and_then(|session| session.known_hosts()) else {
|
||||
return;
|
||||
};
|
||||
load_known_hosts(&mut known_hosts, &known_hosts_files());
|
||||
|
||||
let check = check_known_hosts(&known_hosts, host, port, key);
|
||||
if let Some(warning) = known_hosts_discrepancy(host, check) {
|
||||
log::warn!("{warning}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Known hosts files to consult, user file first (so that new keys are
|
||||
/// accepted because of the user file, mirroring ssh's own ordering)
|
||||
fn known_hosts_files() -> Vec<PathBuf> {
|
||||
@@ -426,14 +587,17 @@ fn authenticate(
|
||||
.into())
|
||||
}
|
||||
|
||||
/// Upload `local` to the remote SFTP `path`, updating `bar` per chunk. The
|
||||
/// remote file is created/truncated; Launchpad's upload queue is
|
||||
/// Upload `local` to the remote SFTP `path` on `host`, updating `bar` per
|
||||
/// chunk. The remote file is created/truncated; Launchpad's upload queue is
|
||||
/// write-only, so failures here mean the upload failed — there is nothing
|
||||
/// to inspect server-side.
|
||||
/// to inspect server-side. Every SFTP call is bounded by the session's
|
||||
/// per-call transfer timeout (see [`SSH_TRANSFER_TIMEOUT`]), so a stalled
|
||||
/// server fails the upload instead of hanging it.
|
||||
pub fn upload_file(
|
||||
sftp: &ssh2::Sftp,
|
||||
local: &Path,
|
||||
remote: &str,
|
||||
host: &str,
|
||||
bar: &ProgressBar,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut local_file =
|
||||
@@ -441,7 +605,7 @@ pub fn upload_file(
|
||||
|
||||
let mut remote_file = sftp
|
||||
.create(Path::new(remote))
|
||||
.map_err(|e| format!("cannot create remote file {remote}: {e}"))?;
|
||||
.map_err(|e| format!("cannot create remote file {remote} on {host}: {e}"))?;
|
||||
|
||||
let mut buf = [0u8; 32 * 1024];
|
||||
loop {
|
||||
@@ -451,7 +615,7 @@ pub fn upload_file(
|
||||
}
|
||||
remote_file
|
||||
.write_all(&buf[..n])
|
||||
.map_err(|e| format!("failed uploading to {remote}: {e}"))?;
|
||||
.map_err(|e| format!("failed uploading to {remote} on {host}: {e}"))?;
|
||||
bar.inc(n as u64);
|
||||
}
|
||||
|
||||
@@ -461,13 +625,25 @@ pub fn upload_file(
|
||||
// truncated remote file as a successful upload. `ssh2::File::write` is
|
||||
// unbuffered (`Write::flush` is a documented no-op) and `close`
|
||||
// finalizes the pending writes server-side, so no flush is needed.
|
||||
remote_file
|
||||
.close()
|
||||
.map_err(|e| format!("failed to close remote file '{remote}' after upload: {e}"))?;
|
||||
remote_file.close().map_err(|e| {
|
||||
format!("failed to close remote file '{remote}' on {host} after upload: {e}")
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove the remote file `remote` on `host` (SFTP remove). Used by the
|
||||
/// best-effort cleanup of an interrupted upload: the queue is write-only,
|
||||
/// so the only remote operation ever needed besides `create` is this one.
|
||||
pub fn remove_file(
|
||||
sftp: &ssh2::Sftp,
|
||||
remote: &str,
|
||||
host: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
sftp.unlink(Path::new(remote))
|
||||
.map_err(|e| format!("cannot remove remote file {remote} on {host}: {e}").into())
|
||||
}
|
||||
|
||||
/// Open the SFTP subsystem on `session`
|
||||
pub fn sftp(session: &Session) -> Result<ssh2::Sftp, Box<dyn std::error::Error>> {
|
||||
session
|
||||
@@ -589,6 +765,99 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
/// A pinned-key host whose known_hosts entries record a DIFFERENT key
|
||||
/// is flagged with a message naming the host; every other check result
|
||||
/// stays silent — the published fingerprint remains authoritative.
|
||||
#[test]
|
||||
fn known_hosts_discrepancy_flags_mismatch_only() {
|
||||
let message = known_hosts_discrepancy("ppa.launchpad.net", CheckResult::Mismatch)
|
||||
.expect("a mismatch under a matching pin must be flagged");
|
||||
assert!(
|
||||
message.contains("ppa.launchpad.net"),
|
||||
"unexpected: {message}"
|
||||
);
|
||||
assert!(message.contains("DIFFERENT"), "unexpected: {message}");
|
||||
assert!(
|
||||
message.contains("published fingerprint"),
|
||||
"unexpected: {message}"
|
||||
);
|
||||
assert!(
|
||||
message.contains("stale"),
|
||||
"the user must be told to clean the entry up: {message}"
|
||||
);
|
||||
|
||||
for quiet in [
|
||||
CheckResult::Match,
|
||||
CheckResult::NotFound,
|
||||
CheckResult::Failure,
|
||||
] {
|
||||
assert!(
|
||||
known_hosts_discrepancy("ppa.launchpad.net", quiet).is_none(),
|
||||
"{quiet:?} must not be flagged"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Known hosts loading decisions: a missing file is the normal
|
||||
/// first-use case and stays silent, a readable file is loaded and its
|
||||
/// entries consulted, and an existing but unreadable file yields a
|
||||
/// warning naming it — the user must learn why their maintained
|
||||
/// entries are not being consulted. Offline: a bare session suffices
|
||||
/// for the known-hosts store.
|
||||
#[test]
|
||||
fn load_known_hosts_warns_only_on_unreadable_files() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut known_hosts = Session::new().unwrap().known_hosts().unwrap();
|
||||
|
||||
// A missing file is silent...
|
||||
let missing = dir.path().join("known_hosts");
|
||||
assert!(load_known_hosts(&mut known_hosts, &[missing]).is_empty());
|
||||
|
||||
// ...a readable one is loaded and consulted...
|
||||
let key = b"a test host key";
|
||||
let loaded = dir.path().join("loaded_known_hosts");
|
||||
fs::write(
|
||||
&loaded,
|
||||
format!("host.example.com ssh-ed25519 {}\n", base64_nopad(key)),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
load_known_hosts(&mut known_hosts, &[loaded]).is_empty(),
|
||||
"a readable file must not warn"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
check_known_hosts(&known_hosts, "host.example.com", 22, key),
|
||||
CheckResult::Match
|
||||
),
|
||||
"the loaded entry must be consulted"
|
||||
);
|
||||
|
||||
// ...and an existing but unreadable one is reported, naming the
|
||||
// file. Environments that read through the mode bits (root) cannot
|
||||
// exercise the unreadable case and are skipped.
|
||||
let locked = dir.path().join("locked_known_hosts");
|
||||
fs::write(&locked, b"stale.example.com ssh-ed25519 c3RhbGU=").unwrap();
|
||||
let mut permissions = fs::metadata(&locked).unwrap().permissions();
|
||||
permissions.set_mode(0o000);
|
||||
fs::set_permissions(&locked, permissions).unwrap();
|
||||
if fs::read(&locked).is_ok() {
|
||||
return;
|
||||
}
|
||||
let warnings = load_known_hosts(&mut known_hosts, &[locked]);
|
||||
assert_eq!(warnings.len(), 1, "unexpected: {warnings:?}");
|
||||
assert!(
|
||||
warnings[0].contains("locked_known_hosts"),
|
||||
"the warning must name the file: {warnings:?}"
|
||||
);
|
||||
assert!(
|
||||
warnings[0].contains("ignored"),
|
||||
"the warning must say the entries are not consulted: {warnings:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Negated `Host` patterns exclude the host from the whole block (the
|
||||
/// OpenSSH rule): a block applies iff at least one positive pattern
|
||||
/// matches AND no negated pattern matches.
|
||||
@@ -761,4 +1030,89 @@ Host other
|
||||
apply_config_file(&mut config, content, host);
|
||||
config
|
||||
}
|
||||
|
||||
/// The all-addresses-failed message names the target and reports every
|
||||
/// attempted address with its error. Pure: no network involved.
|
||||
#[test]
|
||||
fn connect_failed_message_names_target_and_attempts() {
|
||||
let attempts = vec![
|
||||
(
|
||||
SocketAddr::new(
|
||||
std::net::IpAddr::V4(std::net::Ipv4Addr::new(192, 0, 2, 10)),
|
||||
22,
|
||||
),
|
||||
std::io::Error::new(std::io::ErrorKind::TimedOut, "connection timed out"),
|
||||
),
|
||||
(
|
||||
SocketAddr::new(std::net::IpAddr::V6(std::net::Ipv6Addr::LOCALHOST), 22),
|
||||
std::io::Error::new(std::io::ErrorKind::ConnectionRefused, "refused"),
|
||||
),
|
||||
];
|
||||
|
||||
let message = connect_failed_message("ppa.launchpad.net", 22, &attempts);
|
||||
assert!(
|
||||
message.contains("cannot connect to ppa.launchpad.net:22"),
|
||||
"unexpected: {message}"
|
||||
);
|
||||
assert!(
|
||||
message.contains("tried 2 address(es)"),
|
||||
"unexpected: {message}"
|
||||
);
|
||||
assert!(message.contains("192.0.2.10:22"), "unexpected: {message}");
|
||||
assert!(
|
||||
message.contains("connection timed out"),
|
||||
"unexpected: {message}"
|
||||
);
|
||||
assert!(message.contains("[::1]:22"), "unexpected: {message}");
|
||||
assert!(message.contains("refused"), "unexpected: {message}");
|
||||
}
|
||||
|
||||
/// The whole `tcp_connect` failure path on a loopback port with no
|
||||
/// listener (numeric literal, so no DNS; loopback only, so no external
|
||||
/// network): the per-address `connect_timeout` loop produces the
|
||||
/// formatted message naming the target. DNS resolution failures are
|
||||
/// deliberately not tested: resolving a bogus name would touch the
|
||||
/// system resolver, which is not offline-safe.
|
||||
#[test]
|
||||
fn tcp_connect_closed_port_fails_naming_the_target() {
|
||||
// Grab a free port, then release it: connecting to the now-closed
|
||||
// port fails immediately (ECONNREFUSED), never reaching the timeout
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
drop(listener);
|
||||
|
||||
let error = tcp_connect("127.0.0.1", addr.port()).unwrap_err();
|
||||
assert!(
|
||||
error.contains(&format!("cannot connect to 127.0.0.1:{}", addr.port())),
|
||||
"unexpected: {error}"
|
||||
);
|
||||
assert!(error.contains("tried 1 address(es)"), "unexpected: {error}");
|
||||
}
|
||||
|
||||
/// The libssh2 timeout wiring round-trips through the session API with
|
||||
/// the configured constants (no network: a bare session suffices)
|
||||
#[test]
|
||||
fn session_timeouts_use_the_configured_constants() {
|
||||
let session = Session::new().unwrap();
|
||||
|
||||
session.set_timeout(duration_ms(SSH_API_TIMEOUT));
|
||||
assert_eq!(session.timeout(), duration_ms(SSH_API_TIMEOUT));
|
||||
|
||||
// The transfer budget replaces the connection one once authenticated
|
||||
session.set_timeout(duration_ms(SSH_TRANSFER_TIMEOUT));
|
||||
assert_eq!(session.timeout(), duration_ms(SSH_TRANSFER_TIMEOUT));
|
||||
}
|
||||
|
||||
/// Millisecond conversion is exact for the constants and saturates for
|
||||
/// larger values instead of truncating or panicking
|
||||
#[test]
|
||||
fn duration_ms_converts_and_saturates() {
|
||||
assert_eq!(duration_ms(TCP_CONNECT_TIMEOUT), 15_000);
|
||||
assert_eq!(duration_ms(SSH_API_TIMEOUT), 30_000);
|
||||
assert_eq!(duration_ms(SSH_TRANSFER_TIMEOUT), 300_000);
|
||||
assert_eq!(
|
||||
duration_ms(Duration::from_millis(u64::from(u32::MAX) + 1)),
|
||||
u32::MAX
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+2
-6
@@ -379,12 +379,10 @@ fn text_inner(
|
||||
input.push(c);
|
||||
render(Render::Line(&input))?;
|
||||
}
|
||||
event::KeyCode::Backspace => {
|
||||
if !input.is_empty() {
|
||||
event::KeyCode::Backspace if !input.is_empty() => {
|
||||
input.pop();
|
||||
render(Render::Line(&input))?;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
@@ -433,12 +431,10 @@ fn confirm_inner(
|
||||
input.push(c);
|
||||
render(Render::Line(&input))?;
|
||||
}
|
||||
event::KeyCode::Backspace => {
|
||||
if !input.is_empty() {
|
||||
event::KeyCode::Backspace if !input.is_empty() => {
|
||||
input.pop();
|
||||
render(Render::Line(&input))?;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user