Compare commits
10
Commits
2b017dcf43
...
c1f8893576
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c1f8893576 | ||
|
|
e640b153bd | ||
|
|
231c478d0b | ||
|
|
1aa0ca3d2f | ||
|
|
b20acf3199 | ||
|
|
3feca504fc | ||
|
|
85f0d7d92f | ||
|
|
c45edcee76 | ||
|
|
1fc1d1aa77 | ||
|
|
af870cb7cb |
+109
-196
@@ -16,8 +16,6 @@ use std::sync::Arc;
|
|||||||
use crate::context::Context;
|
use crate::context::Context;
|
||||||
use crate::debian::{ChecksumEntry, ControlInfo, FileChecksums, FilesList};
|
use crate::debian::{ChecksumEntry, ControlInfo, FileChecksums, FilesList};
|
||||||
|
|
||||||
use super::parse_checksum_field;
|
|
||||||
|
|
||||||
/// Digests of one artifact.
|
/// Digests of one artifact.
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
struct ArtifactHashes {
|
struct ArtifactHashes {
|
||||||
@@ -57,7 +55,7 @@ pub struct BinaryMetadataOptions {
|
|||||||
/// `dpkg-genchanges -b`: sorted `Binary` list, encounter-order `Architecture`
|
/// `dpkg-genchanges -b`: sorted `Binary` list, encounter-order `Architecture`
|
||||||
/// accumulation, sorted `Description` lines formatted like dpkg, `.buildinfo`
|
/// accumulation, sorted `Description` lines formatted like dpkg, `.buildinfo`
|
||||||
/// registration in `debian/files`, and binary-NMU handling (`Source:
|
/// registration in `debian/files`, and binary-NMU handling (`Source:
|
||||||
/// pkg (prev)` + previous `.dsc` redistribution when present).
|
/// pkg (prev)` + `Binary-Only-Changes`, with no source files distributed).
|
||||||
pub fn generate_binary_metadata(
|
pub fn generate_binary_metadata(
|
||||||
ctx: &Arc<Context>,
|
ctx: &Arc<Context>,
|
||||||
package_dir: &Path,
|
package_dir: &Path,
|
||||||
@@ -137,28 +135,24 @@ pub fn generate_binary_metadata(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ------------------------------------------------------------------
|
// ------------------------------------------------------------------
|
||||||
// Binary-NMU: redistribute the previous source when present
|
// Binary-NMU: reference the previous source version, textually only
|
||||||
// ------------------------------------------------------------------
|
// ------------------------------------------------------------------
|
||||||
let sversion = entry.version.no_epoch();
|
let sversion = entry.version.no_epoch();
|
||||||
let mut source_display = entry.source.clone();
|
let mut source_display = entry.source.clone();
|
||||||
let mut binary_only_changes = None;
|
let mut binary_only_changes = None;
|
||||||
|
|
||||||
if entry.binary_only {
|
if entry.binary_only {
|
||||||
// A binary-only upload must reference the previous source version;
|
// Like dpkg-genchanges/genbuildinfo, a binary-only upload references
|
||||||
// a changelog that cannot yield it is a hard error, like in the
|
// the previous source version in the `Source` field and records the
|
||||||
// source-build path. Reuse the changelog read above instead of
|
// entry in `Binary-Only-Changes`, but distributes NO source files:
|
||||||
// reading the file a second time.
|
// the previous `.dsc` and its tarballs already sit in the archive,
|
||||||
|
// and are not re-uploaded even when present next to the tree.
|
||||||
if let Some(prev) = &previous_entry {
|
if let Some(prev) = &previous_entry {
|
||||||
source_display = format!("{} ({})", entry.source, prev.version.full());
|
source_display = format!("{} ({})", entry.source, prev.version.full());
|
||||||
binary_only_changes = Some(format!(
|
binary_only_changes = Some(format!(
|
||||||
"{}\n\n -- {} <{}> {}",
|
"{}\n\n -- {} <{}> {}",
|
||||||
entry.changes_field, entry.maintainer_name, entry.maintainer_email, entry.date_raw
|
entry.changes_field, entry.maintainer_name, entry.maintainer_email, entry.date_raw
|
||||||
));
|
));
|
||||||
let dsc_name = format!("{}_{}.dsc", entry.source, prev.version.no_epoch());
|
|
||||||
let dsc_path = upload_dir.join(&dsc_name);
|
|
||||||
if ctx.exists(&dsc_path)? {
|
|
||||||
include_dsc_artifacts(ctx, upload_dir, &dsc_name, &mut checksums)?;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -297,6 +291,7 @@ pub fn generate_binary_metadata(
|
|||||||
date: entry.date_raw.clone(),
|
date: entry.date_raw.clone(),
|
||||||
source: source_display,
|
source: source_display,
|
||||||
binaries,
|
binaries,
|
||||||
|
binary_only: entry.binary_only,
|
||||||
built_for_profiles: opts.profiles.clone(),
|
built_for_profiles: opts.profiles.clone(),
|
||||||
architecture: arch_values.join(" "),
|
architecture: arch_values.join(" "),
|
||||||
version: entry.version.full(),
|
version: entry.version.full(),
|
||||||
@@ -400,90 +395,6 @@ fn hashes_in_context(
|
|||||||
Ok(out)
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Pull the `.dsc` checksums (and its referenced tarballs) into the
|
|
||||||
/// checksum registry, mirroring how binary-NMU uploads redistribute the
|
|
||||||
/// previous source.
|
|
||||||
fn include_dsc_artifacts(
|
|
||||||
ctx: &Arc<Context>,
|
|
||||||
upload_dir: &Path,
|
|
||||||
dsc_name: &str,
|
|
||||||
checksums: &mut FileChecksums,
|
|
||||||
) -> Result<(), Box<dyn Error>> {
|
|
||||||
let dsc_content = ctx.read_file(&upload_dir.join(dsc_name))?;
|
|
||||||
let para = crate::debian::control::parse_paragraphs(
|
|
||||||
crate::debian::control::strip_clearsigned_armour(&dsc_content),
|
|
||||||
)
|
|
||||||
.into_iter()
|
|
||||||
.next()
|
|
||||||
.ok_or_else(|| format!("'{dsc_name}' is empty"))?;
|
|
||||||
|
|
||||||
// Names and partial checksums are filled from the very same validated
|
|
||||||
// lines, so a listed name can never miss its checksum entry.
|
|
||||||
// Distribution order follows the Checksums fields (Checksums-Sha1 then
|
|
||||||
// Checksums-Sha256), like dpkg-genchanges; the `Files` field only
|
|
||||||
// supplements the md5 digests.
|
|
||||||
let mut names: Vec<String> = Vec::new();
|
|
||||||
let mut partials: BTreeMap<String, super::PartialChecksum> = BTreeMap::new();
|
|
||||||
for field in ["Checksums-Sha1", "Checksums-Sha256", "Files"] {
|
|
||||||
let Some(value) = para.get(field) else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
for cl in parse_checksum_field(field, value)
|
|
||||||
.map_err(|e| format!("cannot parse '{dsc_name}': {e}"))?
|
|
||||||
{
|
|
||||||
let slot = partials.entry(cl.name.clone()).or_default();
|
|
||||||
match field {
|
|
||||||
"Checksums-Sha1" => slot.sha1 = Some(cl.digest),
|
|
||||||
"Checksums-Sha256" => slot.sha256 = Some(cl.digest),
|
|
||||||
_ => slot.md5 = Some(cl.digest),
|
|
||||||
}
|
|
||||||
slot.size = Some(cl.size);
|
|
||||||
if field != "Files" && !names.contains(&cl.name) {
|
|
||||||
names.push(cl.name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The .dsc itself is hashed fresh (it may be signed/rewritten); the
|
|
||||||
// tarballs reuse the .dsc-recorded digests, like dpkg-genchanges does.
|
|
||||||
let dsc_hashes =
|
|
||||||
hashes_in_context(ctx, upload_dir, std::slice::from_ref(&dsc_name.to_string()))?;
|
|
||||||
if let Some(h) = dsc_hashes.get(dsc_name) {
|
|
||||||
checksums.insert_entry(
|
|
||||||
dsc_name,
|
|
||||||
ChecksumEntry {
|
|
||||||
size: h.size,
|
|
||||||
md5: h.md5.clone(),
|
|
||||||
sha1: h.sha1.clone(),
|
|
||||||
sha256: h.sha256.clone(),
|
|
||||||
// No SHA-512 digest available (see above).
|
|
||||||
sha512: String::new(),
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
for name in &names {
|
|
||||||
if name == dsc_name {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let p = partials
|
|
||||||
.get(name)
|
|
||||||
.ok_or_else(|| format!("file '{name}' listed in '{dsc_name}' has no checksum entry"))?;
|
|
||||||
checksums.insert_entry(
|
|
||||||
name,
|
|
||||||
ChecksumEntry {
|
|
||||||
size: p.size.unwrap_or(0),
|
|
||||||
md5: p.md5.clone().unwrap_or_default(),
|
|
||||||
sha1: p.sha1.clone().unwrap_or_default(),
|
|
||||||
sha256: p.sha256.clone().unwrap_or_default(),
|
|
||||||
// The `.dsc` records no SHA-512 (dpkg only writes
|
|
||||||
// sha1/sha256 there).
|
|
||||||
sha512: String::new(),
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -521,109 +432,11 @@ mod tests {
|
|||||||
assert!(!environment.contains("DEBIAN_FRONTEND"), "{environment}");
|
assert!(!environment.contains("DEBIAN_FRONTEND"), "{environment}");
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A minimal previous-version `.dsc` with a 3-column Checksums-Sha1
|
|
||||||
/// field, a 4-column Checksums-Sha256 line and a 3-column `Files`.
|
|
||||||
/// Regression: the old code filled `names` from any line with a third
|
|
||||||
/// column but `partials` only from exactly-3-column lines, so the
|
|
||||||
/// "bogus" name landed in `names` alone and `&partials["bogus"]`
|
|
||||||
/// panicked. It must produce a build error instead.
|
|
||||||
#[test]
|
|
||||||
fn dsc_four_column_checksum_line_errors_instead_of_panicking() {
|
|
||||||
let dsc_name = "hello_1.0-1.dsc";
|
|
||||||
let dsc = "\
|
|
||||||
Format: 3.0 (native)
|
|
||||||
Source: hello
|
|
||||||
Binary: hello
|
|
||||||
Architecture: any
|
|
||||||
Version: 1.0-1
|
|
||||||
Maintainer: A B <a@b.c>
|
|
||||||
Checksums-Sha1:
|
|
||||||
aaa111 12 hello_1.0.orig.tar.xz
|
|
||||||
Checksums-Sha256:
|
|
||||||
bbb222 12 bogus hello_1.0-1.debian.tar.xz
|
|
||||||
Files:
|
|
||||||
ddd333 12 hello_1.0.orig.tar.xz
|
|
||||||
";
|
|
||||||
let base = tempfile::tempdir().expect("tempdir");
|
|
||||||
std::fs::write(base.path().join(dsc_name), dsc).expect("write dsc");
|
|
||||||
|
|
||||||
let ctx = Arc::new(
|
|
||||||
crate::context::Context::new(crate::context::ContextConfig::Local).expect("context"),
|
|
||||||
);
|
|
||||||
let mut checksums = FileChecksums::new();
|
|
||||||
let err = include_dsc_artifacts(&ctx, base.path(), dsc_name, &mut checksums)
|
|
||||||
.expect_err("malformed Checksums-Sha256 line must fail the build");
|
|
||||||
let err = err.to_string();
|
|
||||||
assert!(err.contains("Checksums-Sha256"), "{err}");
|
|
||||||
assert!(err.contains("hello_1.0-1.debian.tar.xz"), "{err}");
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Happy path: tarball entries are assembled from the Checksums fields
|
|
||||||
/// (sha1/sha256) and merged with the legacy 5-column `Files` md5, in
|
|
||||||
/// Checksums-Sha1 order, with the `.dsc` itself hashed fresh first.
|
|
||||||
#[test]
|
|
||||||
fn include_dsc_artifacts_merges_legacy_files_layout() {
|
|
||||||
let dsc_name = "hello_1.0-1.dsc";
|
|
||||||
let tarball = "hello_1.0.orig.tar.xz";
|
|
||||||
let dsc = "\
|
|
||||||
Format: 3.0 (quilt)
|
|
||||||
Source: hello
|
|
||||||
Binary: hello
|
|
||||||
Architecture: any
|
|
||||||
Version: 1.0-1
|
|
||||||
Maintainer: A B <a@b.c>
|
|
||||||
Checksums-Sha1:
|
|
||||||
aaa111 12 hello_1.0.orig.tar.xz
|
|
||||||
Checksums-Sha256:
|
|
||||||
bbb222 12 hello_1.0.orig.tar.xz
|
|
||||||
Files:
|
|
||||||
ddd333 12 devel optional hello_1.0.orig.tar.xz
|
|
||||||
";
|
|
||||||
let base = tempfile::tempdir().expect("tempdir");
|
|
||||||
std::fs::write(base.path().join(dsc_name), dsc).expect("write dsc");
|
|
||||||
std::fs::write(base.path().join(tarball), "tarball bytes").expect("write tarball");
|
|
||||||
|
|
||||||
let ctx = Arc::new(
|
|
||||||
crate::context::Context::new(crate::context::ContextConfig::Local).expect("context"),
|
|
||||||
);
|
|
||||||
let mut checksums = FileChecksums::new();
|
|
||||||
include_dsc_artifacts(&ctx, base.path(), dsc_name, &mut checksums)
|
|
||||||
.expect("valid dsc must parse");
|
|
||||||
|
|
||||||
let collected: Vec<(String, crate::debian::ChecksumEntry)> = checksums
|
|
||||||
.iter()
|
|
||||||
.map(|(k, e)| (k.clone(), e.clone()))
|
|
||||||
.collect();
|
|
||||||
assert_eq!(
|
|
||||||
collected
|
|
||||||
.iter()
|
|
||||||
.map(|(k, _)| k.as_str())
|
|
||||||
.collect::<Vec<_>>(),
|
|
||||||
vec![dsc_name, tarball],
|
|
||||||
".dsc first, then Checksums-Sha1 order"
|
|
||||||
);
|
|
||||||
|
|
||||||
// The .dsc is hashed fresh from disk.
|
|
||||||
let dsc_entry = &collected[0].1;
|
|
||||||
assert_eq!(dsc_entry.size, dsc.len() as u64);
|
|
||||||
assert_eq!(dsc_entry.md5.len(), 32);
|
|
||||||
assert_eq!(dsc_entry.sha1.len(), 40);
|
|
||||||
assert_eq!(dsc_entry.sha256.len(), 64);
|
|
||||||
|
|
||||||
// The tarball reuses the .dsc-recorded digests, including the
|
|
||||||
// legacy 5-column `Files` md5 (section/priority skipped).
|
|
||||||
let tar_entry = &collected[1].1;
|
|
||||||
assert_eq!(tar_entry.size, 12);
|
|
||||||
assert_eq!(tar_entry.md5, "ddd333");
|
|
||||||
assert_eq!(tar_entry.sha1, "aaa111");
|
|
||||||
assert_eq!(tar_entry.sha256, "bbb222");
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A binary-only (binNMU) build whose changelog cannot yield the
|
/// A binary-only (binNMU) build whose changelog cannot yield the
|
||||||
/// previous entry (malformed second header, unbalanced parenthesis) must
|
/// previous entry (malformed second header, unbalanced parenthesis) must
|
||||||
/// fail the metadata generation with a diagnostic naming the problem,
|
/// fail the metadata generation with a diagnostic naming the problem,
|
||||||
/// instead of silently emitting a plain `Source:` `.changes` with no
|
/// instead of silently emitting a plain `Source:` `.changes` with no
|
||||||
/// `Binary-Only-Changes` and no redistributed previous `.dsc`.
|
/// `Binary-Only-Changes` and no previous-version reference.
|
||||||
#[test]
|
#[test]
|
||||||
fn binary_only_prev_version_parse_failure_errors_instead_of_wrong_metadata() {
|
fn binary_only_prev_version_parse_failure_errors_instead_of_wrong_metadata() {
|
||||||
let changelog = "\
|
let changelog = "\
|
||||||
@@ -739,4 +552,104 @@ Description: test package
|
|||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
assert!(err.contains("Permission denied"), "{err}");
|
assert!(err.contains("Permission denied"), "{err}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A binary-only (binNMU) build references the previous source version
|
||||||
|
/// (`Source: pkg (prev)`, `Binary-Only-Changes`) but must NOT
|
||||||
|
/// redistribute any source file: like dpkg-genchanges/genbuildinfo, the
|
||||||
|
/// previous `.dsc` and its tarballs stay out of both documents even when
|
||||||
|
/// they exist next to the artifacts.
|
||||||
|
#[test]
|
||||||
|
fn binary_only_metadata_references_previous_source_without_redistributing_it() {
|
||||||
|
let changelog = "\
|
||||||
|
hello (1.0-1+b1) unstable; urgency=medium, binary-only=yes
|
||||||
|
|
||||||
|
* Binary-only rebuild.
|
||||||
|
|
||||||
|
-- A B <a@b.c> Mon, 01 Jan 2024 00:00:00 +0000
|
||||||
|
|
||||||
|
hello (1.0-1) unstable; urgency=medium
|
||||||
|
|
||||||
|
* Initial release.
|
||||||
|
|
||||||
|
-- A B <a@b.c> Sun, 31 Dec 2023 00:00:00 +0000
|
||||||
|
";
|
||||||
|
let control = "\
|
||||||
|
Source: hello
|
||||||
|
Section: devel
|
||||||
|
Priority: optional
|
||||||
|
Maintainer: A B <a@b.c>
|
||||||
|
|
||||||
|
Package: hello
|
||||||
|
Architecture: all
|
||||||
|
Description: test package
|
||||||
|
";
|
||||||
|
let base = tempfile::tempdir().expect("tempdir");
|
||||||
|
let tree = base.path().join("hello-1.0");
|
||||||
|
std::fs::create_dir_all(tree.join("debian")).expect("mkdir tree");
|
||||||
|
std::fs::write(tree.join("debian/changelog"), changelog).expect("write changelog");
|
||||||
|
std::fs::write(tree.join("debian/control"), control).expect("write control");
|
||||||
|
std::fs::write(
|
||||||
|
tree.join("debian/files"),
|
||||||
|
"hello_1.0-1+b1_all.deb devel optional\n",
|
||||||
|
)
|
||||||
|
.expect("write files");
|
||||||
|
std::fs::write(base.path().join("hello_1.0-1+b1_all.deb"), "deb payload")
|
||||||
|
.expect("write deb");
|
||||||
|
|
||||||
|
// The trap: the previous source artifacts sit right next to the
|
||||||
|
// binaries, as they would after a source build. dpkg does not
|
||||||
|
// redistribute them for a binary-only upload, and neither must we.
|
||||||
|
std::fs::write(
|
||||||
|
base.path().join("hello_1.0-1.dsc"),
|
||||||
|
"Format: 3.0 (quilt)\nSource: hello\nBinary: hello\nArchitecture: any\nVersion: \
|
||||||
|
1.0-1\nMaintainer: A B <a@b.c>\nChecksums-Sha1:\n aaa111 12 \
|
||||||
|
hello_1.0.orig.tar.xz\n",
|
||||||
|
)
|
||||||
|
.expect("write previous dsc");
|
||||||
|
std::fs::write(base.path().join("hello_1.0.orig.tar.xz"), "tarball bytes")
|
||||||
|
.expect("write previous tarball");
|
||||||
|
|
||||||
|
let ctx = Arc::new(
|
||||||
|
crate::context::Context::new(crate::context::ContextConfig::Local).expect("context"),
|
||||||
|
);
|
||||||
|
let opts = BinaryMetadataOptions {
|
||||||
|
profiles: Vec::new(),
|
||||||
|
vendor: "debian".to_string(),
|
||||||
|
exported_env: BTreeMap::new(),
|
||||||
|
build_arch: "amd64".to_string(),
|
||||||
|
host_arch: "amd64".to_string(),
|
||||||
|
};
|
||||||
|
let (buildinfo_path, changes_path) =
|
||||||
|
generate_binary_metadata(&ctx, &tree, base.path(), &opts)
|
||||||
|
.expect("binNMU metadata generation must succeed");
|
||||||
|
|
||||||
|
let changes = std::fs::read_to_string(&changes_path).expect("read changes");
|
||||||
|
let buildinfo = std::fs::read_to_string(&buildinfo_path).expect("read buildinfo");
|
||||||
|
|
||||||
|
// The previous version is referenced textually.
|
||||||
|
assert!(
|
||||||
|
changes.contains("Source: hello (1.0-1)"),
|
||||||
|
"changes must reference the previous version: {changes}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
buildinfo.contains("Binary-Only-Changes"),
|
||||||
|
"buildinfo must record the binary-only entry: {buildinfo}"
|
||||||
|
);
|
||||||
|
// ... but no source file is distributed, on either side.
|
||||||
|
for (doc, text) in [("changes", &changes), ("buildinfo", &buildinfo)] {
|
||||||
|
assert!(
|
||||||
|
!text.contains("hello_1.0-1.dsc"),
|
||||||
|
"{doc} must not redistribute the previous .dsc: {text}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!text.contains("hello_1.0.orig.tar.xz"),
|
||||||
|
"{doc} must not redistribute the previous tarball: {text}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// The distributed set is exactly the binary artifacts + buildinfo.
|
||||||
|
assert!(
|
||||||
|
changes.contains("hello_1.0-1+b1_all.deb") && changes.contains(".buildinfo"),
|
||||||
|
"changes must distribute the deb and the buildinfo: {changes}"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -65,6 +65,9 @@ pub struct ChangesInput {
|
|||||||
pub source: String,
|
pub source: String,
|
||||||
/// Sorted binary package names with artifacts (empty for source-only).
|
/// Sorted binary package names with artifacts (empty for source-only).
|
||||||
pub binaries: Vec<String>,
|
pub binaries: Vec<String>,
|
||||||
|
/// Whether the changelog entry is a binary-only (binNMU) upload
|
||||||
|
/// (`Binary-Only: yes` field).
|
||||||
|
pub binary_only: bool,
|
||||||
/// Active build profiles (`Built-For-Profiles`); omitted when empty.
|
/// Active build profiles (`Built-For-Profiles`); omitted when empty.
|
||||||
pub built_for_profiles: Vec<String>,
|
pub built_for_profiles: Vec<String>,
|
||||||
/// `Architecture` field value in encounter order (e.g. `source`,
|
/// `Architecture` field value in encounter order (e.g. `source`,
|
||||||
@@ -142,6 +145,9 @@ pub fn render_changes(input: &ChangesInput) -> Paragraph {
|
|||||||
let joined = input.binaries.join(" ");
|
let joined = input.binaries.join(" ");
|
||||||
p.set("Binary", &wrap_long(&joined));
|
p.set("Binary", &wrap_long(&joined));
|
||||||
}
|
}
|
||||||
|
if input.binary_only {
|
||||||
|
p.set("Binary-Only", "yes");
|
||||||
|
}
|
||||||
if !input.built_for_profiles.is_empty() {
|
if !input.built_for_profiles.is_empty() {
|
||||||
p.set("Built-For-Profiles", &input.built_for_profiles.join(" "));
|
p.set("Built-For-Profiles", &input.built_for_profiles.join(" "));
|
||||||
}
|
}
|
||||||
@@ -329,6 +335,7 @@ mod tests {
|
|||||||
date: "Sat, 22 Aug 2026 10:00:00 +0000".to_string(),
|
date: "Sat, 22 Aug 2026 10:00:00 +0000".to_string(),
|
||||||
source: "pkg".to_string(),
|
source: "pkg".to_string(),
|
||||||
binaries: vec![],
|
binaries: vec![],
|
||||||
|
binary_only: false,
|
||||||
built_for_profiles: vec![],
|
built_for_profiles: vec![],
|
||||||
architecture: "source".to_string(),
|
architecture: "source".to_string(),
|
||||||
version: "1.0".to_string(),
|
version: "1.0".to_string(),
|
||||||
|
|||||||
+197
-87
@@ -224,7 +224,8 @@ fn retry_after_revendor(
|
|||||||
/// Run the full native source-build pipeline in `cwd`.
|
/// Run the full native source-build pipeline in `cwd`.
|
||||||
///
|
///
|
||||||
/// Steps (mirroring `dpkg-buildpackage -S -I -i -nc -d`):
|
/// Steps (mirroring `dpkg-buildpackage -S -I -i -nc -d`):
|
||||||
/// 1. sanity checks and metadata resolution (changelog, control),
|
/// 1. sanity checks and metadata resolution (changelog, control); a
|
||||||
|
/// `binary-only=yes` changelog entry is refused, like `dpkg-source -b`,
|
||||||
/// 2. environment setup (`SOURCE_DATE_EPOCH`, `DEB_BUILD_OPTIONS`, arch vars),
|
/// 2. environment setup (`SOURCE_DATE_EPOCH`, `DEB_BUILD_OPTIONS`, arch vars),
|
||||||
/// 3. signing decision (key discovery, UNRELEASED handling),
|
/// 3. signing decision (key discovery, UNRELEASED handling),
|
||||||
/// 4. `dpkg-source --before-build` then `dpkg-source -b`,
|
/// 4. `dpkg-source --before-build` then `dpkg-source -b`,
|
||||||
@@ -281,33 +282,25 @@ pub fn run_source_build(
|
|||||||
let mut entries = crate::debian::changelog::parse_changelog_entries(&changelog_path, Some(2))?;
|
let mut entries = crate::debian::changelog::parse_changelog_entries(&changelog_path, Some(2))?;
|
||||||
let entry = entries.remove(0);
|
let entry = entries.remove(0);
|
||||||
let previous_entry = entries.into_iter().next();
|
let previous_entry = entries.into_iter().next();
|
||||||
|
|
||||||
|
// A binary-only (binNMU) changelog entry is a binary publication whose
|
||||||
|
// source is already in the archive: like dpkg-source, refuse to build
|
||||||
|
// source for it instead of producing binNMU-style source metadata.
|
||||||
|
if entry.binary_only {
|
||||||
|
return Err(
|
||||||
|
"cannot build source for a binary-only publication: the changelog \
|
||||||
|
entry sets binary-only=yes (dpkg-source refuses it too)"
|
||||||
|
.into(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
let ctrl = ControlInfo::parse(&control_path)?;
|
let ctrl = ControlInfo::parse(&control_path)?;
|
||||||
|
|
||||||
if let Some(u) = &ui {
|
if let Some(u) = &ui {
|
||||||
u.set_build_target(&entry.source, &entry.version.full(), &entry.distribution);
|
u.set_build_target(&entry.source, &entry.version.full(), &entry.distribution);
|
||||||
}
|
}
|
||||||
|
|
||||||
// binNMU builds reference the *previous* (source) version in their
|
let source_display = entry.source.clone();
|
||||||
// artifact metadata, like dpkg-genchanges/genbuildinfo do.
|
|
||||||
let previous_version = if entry.binary_only {
|
|
||||||
previous_entry.as_ref().map(|e| e.version.full())
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
let source_display = if entry.binary_only {
|
|
||||||
match previous_version.as_deref() {
|
|
||||||
Some(prev) => format!("{} ({})", entry.source, prev),
|
|
||||||
None => entry.source.clone(),
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
entry.source.clone()
|
|
||||||
};
|
|
||||||
let binary_only_changes = entry.binary_only.then(|| {
|
|
||||||
format!(
|
|
||||||
"{}\n\n -- {} <{}> {}",
|
|
||||||
entry.changes_field, entry.maintainer_name, entry.maintainer_email, entry.date_raw
|
|
||||||
)
|
|
||||||
});
|
|
||||||
|
|
||||||
let sversion = entry.version.no_epoch();
|
let sversion = entry.version.no_epoch();
|
||||||
let dsc_name = format!("{}_{}.dsc", entry.source, sversion);
|
let dsc_name = format!("{}_{}.dsc", entry.source, sversion);
|
||||||
@@ -431,35 +424,16 @@ pub fn run_source_build(
|
|||||||
.into());
|
.into());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Binary-only uploads redistribute the *previous* source: metadata
|
|
||||||
// references the previous version's .dsc (which must already exist in
|
|
||||||
// the output directory), exactly like dpkg-genchanges/genbuildinfo.
|
|
||||||
let ref_dsc_name = match previous_version.as_deref().filter(|_| entry.binary_only) {
|
|
||||||
Some(prev) => {
|
|
||||||
let prev_version = crate::debian::DebianVersion::parse(prev)?;
|
|
||||||
let name = format!("{}_{}.dsc", entry.source, prev_version.no_epoch());
|
|
||||||
if !parent.join(&name).exists() {
|
|
||||||
return Err(format!(
|
|
||||||
"binary-only build requires the previous source '{} \
|
|
||||||
{}' to exist next to the package",
|
|
||||||
entry.source, prev
|
|
||||||
)
|
|
||||||
.into());
|
|
||||||
}
|
|
||||||
name
|
|
||||||
}
|
|
||||||
None => dsc_name.clone(),
|
|
||||||
};
|
|
||||||
let ref_dsc_path = parent.join(&ref_dsc_name);
|
|
||||||
|
|
||||||
// ------------------------------------------------------------------
|
// ------------------------------------------------------------------
|
||||||
// 6. .buildinfo generation (native dpkg-genbuildinfo equivalent)
|
// 6. .buildinfo generation (native dpkg-genbuildinfo equivalent)
|
||||||
// ------------------------------------------------------------------
|
// ------------------------------------------------------------------
|
||||||
if let Some(u) = &ui {
|
if let Some(u) = &ui {
|
||||||
u.progress_message("Generating .buildinfo");
|
u.progress_message("Generating .buildinfo");
|
||||||
}
|
}
|
||||||
let mut checksums = FileChecksums::new();
|
// What the .buildinfo itself records: like dpkg-genbuildinfo, only the
|
||||||
checksums.add_file_as(&ref_dsc_path, &ref_dsc_name)?;
|
// referenced .dsc — not the tarballs, and never the buildinfo itself.
|
||||||
|
let mut buildinfo_checksums = FileChecksums::new();
|
||||||
|
buildinfo_checksums.add_file_as(&dsc_path, &dsc_name)?;
|
||||||
|
|
||||||
let status_path = PathBuf::from("/var/lib/dpkg/status");
|
let status_path = PathBuf::from("/var/lib/dpkg/status");
|
||||||
let bd_fields = [ctrl.source.get("Build-Depends").unwrap_or("")];
|
let bd_fields = [ctrl.source.get("Build-Depends").unwrap_or("")];
|
||||||
@@ -472,7 +446,9 @@ pub fn run_source_build(
|
|||||||
binaries: Vec::new(), // source-only build
|
binaries: Vec::new(), // source-only build
|
||||||
architecture: "source".to_string(),
|
architecture: "source".to_string(),
|
||||||
version: entry.version.full(),
|
version: entry.version.full(),
|
||||||
binary_only_changes: binary_only_changes.clone(),
|
// A binary-only entry never reaches a source build (refused
|
||||||
|
// above), so the buildinfo never carries Binary-Only-Changes.
|
||||||
|
binary_only_changes: None,
|
||||||
build_origin: vendor.clone(),
|
build_origin: vendor.clone(),
|
||||||
build_architecture: arch_vars
|
build_architecture: arch_vars
|
||||||
.get("DEB_BUILD_ARCH")
|
.get("DEB_BUILD_ARCH")
|
||||||
@@ -484,7 +460,7 @@ pub fn run_source_build(
|
|||||||
environment: environment.clone(),
|
environment: environment.clone(),
|
||||||
})
|
})
|
||||||
};
|
};
|
||||||
buildinfo::save_buildinfo(&buildinfo_path, &render_buildinfo_doc(&checksums))?;
|
buildinfo::save_buildinfo(&buildinfo_path, &render_buildinfo_doc(&buildinfo_checksums))?;
|
||||||
|
|
||||||
// Register the .buildinfo in debian/files (as dpkg-genbuildinfo does).
|
// Register the .buildinfo in debian/files (as dpkg-genbuildinfo does).
|
||||||
let files_path = cwd.join("debian/files");
|
let files_path = cwd.join("debian/files");
|
||||||
@@ -506,17 +482,21 @@ pub fn run_source_build(
|
|||||||
if let Some(u) = &ui {
|
if let Some(u) = &ui {
|
||||||
u.progress_message("Generating .changes");
|
u.progress_message("Generating .changes");
|
||||||
}
|
}
|
||||||
|
// What the .changes distributes: the .dsc (with its recorded digests),
|
||||||
|
// the tarballs listed in it (below), and the .buildinfo (last, as
|
||||||
|
// dpkg-genchanges does when it consumes debian/files).
|
||||||
|
let mut checksums = buildinfo_checksums.clone();
|
||||||
// Pull the tarball checksums out of the referenced .dsc so they are
|
// Pull the tarball checksums out of the referenced .dsc so they are
|
||||||
// distributed through the .changes like dpkg-genchanges does, in the
|
// distributed through the .changes like dpkg-genchanges does, in the
|
||||||
// order the .dsc itself lists them.
|
// order the .dsc itself lists them.
|
||||||
let dsc_content = std::fs::read_to_string(&ref_dsc_path)
|
let dsc_content = std::fs::read_to_string(&dsc_path)
|
||||||
.map_err(|e| format!("cannot read '{}': {}", ref_dsc_path.display(), e))?;
|
.map_err(|e| format!("cannot read '{}': {}", dsc_path.display(), e))?;
|
||||||
let dsc_para = parse_paragraphs(crate::debian::control::strip_clearsigned_armour(
|
let dsc_para = parse_paragraphs(crate::debian::control::strip_clearsigned_armour(
|
||||||
&dsc_content,
|
&dsc_content,
|
||||||
))
|
))
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.next()
|
.next()
|
||||||
.ok_or_else(|| format!("'{}' is empty", ref_dsc_path.display()))?;
|
.ok_or_else(|| format!("'{}' is empty", dsc_path.display()))?;
|
||||||
|
|
||||||
// Whether the upload redistributes the upstream tarballs (dpkg
|
// Whether the upload redistributes the upstream tarballs (dpkg
|
||||||
// -sa/-si/-sd source styles). Stripping only applies to a split source
|
// -sa/-si/-sd source styles). Stripping only applies to a split source
|
||||||
@@ -536,7 +516,7 @@ pub fn run_source_build(
|
|||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
for cl in parse_checksum_field(field, value)
|
for cl in parse_checksum_field(field, value)
|
||||||
.map_err(|e| format!("cannot parse '{}': {e}", ref_dsc_path.display()))?
|
.map_err(|e| format!("cannot parse '{}': {e}", dsc_path.display()))?
|
||||||
{
|
{
|
||||||
if !dsc_files.contains_key(&cl.name) {
|
if !dsc_files.contains_key(&cl.name) {
|
||||||
dsc_file_names.push(cl.name.clone());
|
dsc_file_names.push(cl.name.clone());
|
||||||
@@ -576,7 +556,7 @@ pub fn run_source_build(
|
|||||||
};
|
};
|
||||||
|
|
||||||
for name in &dsc_file_names {
|
for name in &dsc_file_names {
|
||||||
if name == &ref_dsc_name {
|
if name == &dsc_name {
|
||||||
continue; // already computed directly above
|
continue; // already computed directly above
|
||||||
}
|
}
|
||||||
if is_stripped(name) {
|
if is_stripped(name) {
|
||||||
@@ -594,7 +574,7 @@ pub fn run_source_build(
|
|||||||
let partial = dsc_files.get(name).ok_or_else(|| {
|
let partial = dsc_files.get(name).ok_or_else(|| {
|
||||||
format!(
|
format!(
|
||||||
"file '{name}' listed in '{}' has no checksum entry",
|
"file '{name}' listed in '{}' has no checksum entry",
|
||||||
ref_dsc_path.display()
|
dsc_path.display()
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
checksums.insert_entry(
|
checksums.insert_entry(
|
||||||
@@ -621,13 +601,9 @@ pub fn run_source_build(
|
|||||||
// dsc and tarballs use the source stanza defaults (not persisted into
|
// dsc and tarballs use the source stanza defaults (not persisted into
|
||||||
// debian/files, matching dpkg).
|
// debian/files, matching dpkg).
|
||||||
let mut changes_files = files_list.clone();
|
let mut changes_files = files_list.clone();
|
||||||
changes_files.add(FilesEntry::new(
|
changes_files.add(FilesEntry::new(&dsc_name, ctrl.section(), ctrl.priority()));
|
||||||
&ref_dsc_name,
|
|
||||||
ctrl.section(),
|
|
||||||
ctrl.priority(),
|
|
||||||
));
|
|
||||||
for name in &dsc_file_names {
|
for name in &dsc_file_names {
|
||||||
if name != &ref_dsc_name && !is_stripped(name) {
|
if name != &dsc_name && !is_stripped(name) {
|
||||||
changes_files.add(FilesEntry::new(name, ctrl.section(), ctrl.priority()));
|
changes_files.add(FilesEntry::new(name, ctrl.section(), ctrl.priority()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -638,6 +614,7 @@ pub fn run_source_build(
|
|||||||
date: entry.date_raw.clone(),
|
date: entry.date_raw.clone(),
|
||||||
source: source_display.clone(),
|
source: source_display.clone(),
|
||||||
binaries: Vec::new(), // source-only upload
|
binaries: Vec::new(), // source-only upload
|
||||||
|
binary_only: entry.binary_only,
|
||||||
built_for_profiles: profiles.clone(),
|
built_for_profiles: profiles.clone(),
|
||||||
architecture: "source".to_string(),
|
architecture: "source".to_string(),
|
||||||
version: entry.version.full(),
|
version: entry.version.full(),
|
||||||
@@ -681,18 +658,18 @@ pub fn run_source_build(
|
|||||||
|
|
||||||
log::info!("Signing {}", dsc_name);
|
log::info!("Signing {}", dsc_name);
|
||||||
crate::utils::gpg::clearsign_file(&dsc_path, &keyid)?;
|
crate::utils::gpg::clearsign_file(&dsc_path, &keyid)?;
|
||||||
// The freshly built .dsc changed: refresh its checksums inside the
|
// The freshly built .dsc changed: refresh its digests in both
|
||||||
// .buildinfo. For binary-only builds the metadata references the
|
// checksum sets.
|
||||||
// *previous* .dsc (untouched by this build), so there is nothing to
|
buildinfo_checksums.add_file_as(&dsc_path, &dsc_name)?;
|
||||||
// refresh.
|
|
||||||
if !entry.binary_only {
|
|
||||||
checksums.add_file_as(&dsc_path, &dsc_name)?;
|
checksums.add_file_as(&dsc_path, &dsc_name)?;
|
||||||
}
|
// Re-render the .buildinfo from its own set (the .dsc only, like
|
||||||
buildinfo::save_buildinfo(&buildinfo_path, &render_buildinfo_doc(&checksums))?;
|
// dpkg-genbuildinfo): it must not list the tarballs or itself.
|
||||||
|
buildinfo::save_buildinfo(&buildinfo_path, &render_buildinfo_doc(&buildinfo_checksums))?;
|
||||||
|
|
||||||
log::info!("Signing {}", buildinfo_name);
|
log::info!("Signing {}", buildinfo_name);
|
||||||
crate::utils::gpg::clearsign_file(&buildinfo_path, &keyid)?;
|
crate::utils::gpg::clearsign_file(&buildinfo_path, &keyid)?;
|
||||||
// Both .dsc and .buildinfo changed: refresh the .changes.
|
// Both .dsc and .buildinfo changed: refresh the .changes with the
|
||||||
|
// signed buildinfo's fresh digests.
|
||||||
checksums.add_file_as(&buildinfo_path, &buildinfo_name)?;
|
checksums.add_file_as(&buildinfo_path, &buildinfo_name)?;
|
||||||
changes::save_changes(&changes_path, &render_changes_doc(&checksums))?;
|
changes::save_changes(&changes_path, &render_changes_doc(&checksums))?;
|
||||||
|
|
||||||
@@ -1040,6 +1017,35 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::debian::DebianVersion;
|
use crate::debian::DebianVersion;
|
||||||
|
|
||||||
|
/// A binary-only (binNMU) changelog entry is a binary publication: like
|
||||||
|
/// `dpkg-source -b`, the source build must refuse it outright instead of
|
||||||
|
/// producing binNMU-style source metadata referencing the previous
|
||||||
|
/// version's `.dsc`.
|
||||||
|
#[test]
|
||||||
|
fn source_build_refuses_binary_only_changelog() {
|
||||||
|
let base = tempfile::tempdir().expect("tempdir");
|
||||||
|
let tree = base.path().join("hello-1.0");
|
||||||
|
std::fs::create_dir_all(tree.join("debian")).expect("mkdir tree");
|
||||||
|
std::fs::write(
|
||||||
|
tree.join("debian/changelog"),
|
||||||
|
"hello (1.0-1+b1) unstable; urgency=medium, binary-only=yes\n\n \
|
||||||
|
* Binary-only rebuild.\n\n -- A B <a@b.c> Mon, 01 Jan 2024 00:00:00 +0000\n",
|
||||||
|
)
|
||||||
|
.expect("write changelog");
|
||||||
|
std::fs::write(
|
||||||
|
tree.join("debian/control"),
|
||||||
|
"Source: hello\nMaintainer: A B <a@b.c>\n",
|
||||||
|
)
|
||||||
|
.expect("write control");
|
||||||
|
std::fs::write(tree.join("debian/rules"), "#!/usr/bin/make -f\n").expect("write rules");
|
||||||
|
|
||||||
|
let err = run_source_build(&tree, &SourceBuildOptions::default(), None)
|
||||||
|
.expect_err("binary-only entries must not build a source package");
|
||||||
|
let err = err.to_string();
|
||||||
|
assert!(err.contains("binary-only"), "{err}");
|
||||||
|
assert!(err.contains("source"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn partial_checksum_defaults() {
|
fn partial_checksum_defaults() {
|
||||||
let p = PartialChecksum::default();
|
let p = PartialChecksum::default();
|
||||||
@@ -1266,6 +1272,8 @@ mod differential_tests {
|
|||||||
extra_source_fields: &'static [(&'static str, &'static str)],
|
extra_source_fields: &'static [(&'static str, &'static str)],
|
||||||
/// Version of the previous changelog entry, when the fixture has one.
|
/// Version of the previous changelog entry, when the fixture has one.
|
||||||
previous_version: Option<&'static str>,
|
previous_version: Option<&'static str>,
|
||||||
|
/// Mark the newest changelog entry `binary-only=yes` (binNMU).
|
||||||
|
binary_only_marker: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FixtureSpec {
|
impl FixtureSpec {
|
||||||
@@ -1283,6 +1291,7 @@ mod differential_tests {
|
|||||||
patches: &[],
|
patches: &[],
|
||||||
extra_source_fields: &[],
|
extra_source_fields: &[],
|
||||||
previous_version: None,
|
previous_version: None,
|
||||||
|
binary_only_marker: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1295,9 +1304,14 @@ mod differential_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn changelog(&self) -> String {
|
fn changelog(&self) -> String {
|
||||||
|
let params = if self.binary_only_marker {
|
||||||
|
format!("urgency={}, binary-only=yes", self.urgency)
|
||||||
|
} else {
|
||||||
|
format!("urgency={}", self.urgency)
|
||||||
|
};
|
||||||
let mut out = format!(
|
let mut out = format!(
|
||||||
"{} ({}) {}; urgency={}\n\n",
|
"{} ({}) {}; {}\n\n",
|
||||||
self.name, self.version, self.distribution, self.urgency
|
self.name, self.version, self.distribution, params
|
||||||
);
|
);
|
||||||
for line in self.body {
|
for line in self.body {
|
||||||
out.push_str(" * ");
|
out.push_str(" * ");
|
||||||
@@ -1863,23 +1877,50 @@ Provides: virtual-thing (= 2.0), plain-virtual
|
|||||||
#[test]
|
#[test]
|
||||||
fn diff_binary_build_metadata() {
|
fn diff_binary_build_metadata() {
|
||||||
const NAME: &str = "pkh-diff-m";
|
const NAME: &str = "pkh-diff-m";
|
||||||
let control = format!(
|
|
||||||
"Source: {NAME}\nSection: utils\nPriority: optional\nMaintainer: {MAINTAINER}\nBuild-Depends: libc6\n\n\
|
|
||||||
Package: {NAME}\nArchitecture: any\nDescription: test package main\n long description\n\n\
|
|
||||||
Package: {NAME}-u\nPackage-Type: udeb\nArchitecture: all\nDescription: test udeb\n short\n"
|
|
||||||
);
|
|
||||||
let changelog = format!(
|
let changelog = format!(
|
||||||
"{NAME} (1.0-1) unstable; urgency=medium\n\n * Binary build test.\n\n -- {MAINTAINER} {DATE}\n"
|
"{NAME} (1.0-1) unstable; urgency=medium\n\n * Binary build test.\n\n -- {MAINTAINER} {DATE}\n"
|
||||||
);
|
);
|
||||||
let rules = format!(
|
diff_binary_metadata_case(NAME, &changelog, "1.0-1", false);
|
||||||
"#!/usr/bin/make -f\nV = $(shell dpkg-parsechangelog -S Version)\nA = $(shell dpkg-architecture -qDEB_HOST_ARCH)\n\nbuild:\n\tmkdir -p debian/tmp/usr/bin\n\tprintf '#!/bin/sh\\necho hi\\n' > debian/tmp/usr/bin/hello\n\tchmod 755 debian/tmp/usr/bin/hello\n\ttouch $@\n\nbinary: build\n\trm -rf debian/{NAME} debian/{NAME}-u\n\tmkdir -p debian/{NAME}/usr/bin debian/{NAME}/DEBIAN\n\tcp -r debian/tmp/. debian/{NAME}/\n\tdpkg-gencontrol -p{NAME} -Pdebian/{NAME}\n\tdpkg-deb --build debian/{NAME} ..\n\tmkdir -p debian/{NAME}-u/usr/share debian/{NAME}-u/DEBIAN\n\techo data > debian/{NAME}-u/usr/share/data.txt\n\tdpkg-gencontrol -p{NAME}-u -Pdebian/{NAME}-u\n\tdpkg-deb --build debian/{NAME}-u ..\n\tmv ../{NAME}-u_$(V)_all.deb ../{NAME}-u_$(V)_all.udeb\n\nclean:\n\trm -rf debian/tmp debian/{NAME} debian/{NAME}-u build-stamp debian/files debian/*.substvars\n\n.PHONY: build binary clean\n"
|
}
|
||||||
);
|
|
||||||
|
/// debian/control shared by the binary-metadata differential cases: one
|
||||||
|
/// arch:any deb and one arch:all udeb.
|
||||||
|
fn binary_test_control(name: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"Source: {name}\nSection: utils\nPriority: optional\nMaintainer: {MAINTAINER}\nBuild-Depends: libc6\n\n\
|
||||||
|
Package: {name}\nArchitecture: any\nDescription: test package main\n long description\n\n\
|
||||||
|
Package: {name}-u\nPackage-Type: udeb\nArchitecture: all\nDescription: test udeb\n short\n"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// debian/rules driving dpkg-gencontrol/dpkg-deb directly (no debhelper).
|
||||||
|
fn binary_test_rules(name: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"#!/usr/bin/make -f\nV = $(shell dpkg-parsechangelog -S Version)\nA = $(shell dpkg-architecture -qDEB_HOST_ARCH)\n\nbuild:\n\tmkdir -p debian/tmp/usr/bin\n\tprintf '#!/bin/sh\\necho hi\\n' > debian/tmp/usr/bin/hello\n\tchmod 755 debian/tmp/usr/bin/hello\n\ttouch $@\n\nbinary: build\n\trm -rf debian/{name} debian/{name}-u\n\tmkdir -p debian/{name}/usr/bin debian/{name}/DEBIAN\n\tcp -r debian/tmp/. debian/{name}/\n\tdpkg-gencontrol -p{name} -Pdebian/{name}\n\tdpkg-deb --build debian/{name} ..\n\tmkdir -p debian/{name}-u/usr/share debian/{name}-u/DEBIAN\n\techo data > debian/{name}-u/usr/share/data.txt\n\tdpkg-gencontrol -p{name}-u -Pdebian/{name}-u\n\tdpkg-deb --build debian/{name}-u ..\n\tmv ../{name}-u_$(V)_all.deb ../{name}-u_$(V)_all.udeb\n\nclean:\n\trm -rf debian/tmp debian/{name} debian/{name}-u build-stamp debian/files debian/*.substvars\n\n.PHONY: build binary clean\n"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Both-sides binary metadata comparison for one changelog: golden
|
||||||
|
/// `dpkg-buildpackage -b` against the pkh deb flow (rules targets with a
|
||||||
|
/// dpkg-buildpackage-like environment) plus native metadata generation.
|
||||||
|
/// `artifact_version` is the full version the artifacts are named after;
|
||||||
|
/// `with_prev_source` additionally places the previous version's `.dsc`
|
||||||
|
/// and tarball next to the tree on both sides (the binNMU trap: they must
|
||||||
|
/// not be redistributed).
|
||||||
|
fn diff_binary_metadata_case(
|
||||||
|
name: &str,
|
||||||
|
changelog: &str,
|
||||||
|
artifact_version: &str,
|
||||||
|
with_prev_source: bool,
|
||||||
|
) {
|
||||||
|
let control = binary_test_control(name);
|
||||||
|
let rules = binary_test_rules(name);
|
||||||
|
|
||||||
let write_tree = |root: &Path| {
|
let write_tree = |root: &Path| {
|
||||||
fs::create_dir_all(root.join(format!("{NAME}/debian/source"))).expect("mkdir tree");
|
fs::create_dir_all(root.join(format!("{name}/debian/source"))).expect("mkdir tree");
|
||||||
let tree = root.join(NAME);
|
let tree = root.join(name);
|
||||||
fs::write(tree.join("debian/control"), &control).expect("write control");
|
fs::write(tree.join("debian/control"), &control).expect("write control");
|
||||||
fs::write(tree.join("debian/changelog"), &changelog).expect("write changelog");
|
fs::write(tree.join("debian/changelog"), changelog).expect("write changelog");
|
||||||
fs::write(tree.join("debian/source/format"), "3.0 (native)\n").expect("write format");
|
fs::write(tree.join("debian/source/format"), "3.0 (native)\n").expect("write format");
|
||||||
fs::write(tree.join("debian/rules"), &rules).expect("write rules");
|
fs::write(tree.join("debian/rules"), &rules).expect("write rules");
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
@@ -1900,6 +1941,20 @@ Provides: virtual-thing (= 2.0), plain-virtual
|
|||||||
let golden_tree = write_tree(&golden_root);
|
let golden_tree = write_tree(&golden_root);
|
||||||
let ours_tree = write_tree(&ours_root);
|
let ours_tree = write_tree(&ours_root);
|
||||||
|
|
||||||
|
if with_prev_source {
|
||||||
|
for root in [&golden_root, &ours_root] {
|
||||||
|
let dsc = format!(
|
||||||
|
"Format: 3.0 (native)\nSource: {name}\nBinary: {name}\nArchitecture: all\n\
|
||||||
|
Version: 1.0-1\nMaintainer: {MAINTAINER}\nChecksums-Sha1:\n aaa111 12 \
|
||||||
|
{name}_1.0.tar.xz\nChecksums-Sha256:\n bbb222 12 {name}_1.0.tar.xz\nFiles:\n \
|
||||||
|
ddd333 12 utils optional {name}_1.0.tar.xz\n"
|
||||||
|
);
|
||||||
|
fs::write(root.join(format!("{name}_1.0-1.dsc")), dsc).expect("write previous dsc");
|
||||||
|
fs::write(root.join(format!("{name}_1.0.tar.xz")), "tarball byte")
|
||||||
|
.expect("write previous tarball");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Golden side: real dpkg-buildpackage binary build.
|
// Golden side: real dpkg-buildpackage binary build.
|
||||||
let status = crate::test_support::run_logged(
|
let status = crate::test_support::run_logged(
|
||||||
Command::new("dpkg-buildpackage")
|
Command::new("dpkg-buildpackage")
|
||||||
@@ -1915,7 +1970,7 @@ Provides: virtual-thing (= 2.0), plain-virtual
|
|||||||
// rules targets directly by default (missing Rules-Requires-Root is
|
// rules targets directly by default (missing Rules-Requires-Root is
|
||||||
// treated as 'no'), so no fakeroot wrapper here either.
|
// treated as 'no'), so no fakeroot wrapper here either.
|
||||||
let entry =
|
let entry =
|
||||||
crate::debian::parse_changelog_entry_from_str(&changelog).expect("parse changelog");
|
crate::debian::parse_changelog_entry_from_str(changelog).expect("parse changelog");
|
||||||
let vendor = env::current_vendor();
|
let vendor = env::current_vendor();
|
||||||
let profiles = env::resolve_build_profiles(&[], &vendor);
|
let profiles = env::resolve_build_profiles(&[], &vendor);
|
||||||
let parallel = env::num_parallel();
|
let parallel = env::num_parallel();
|
||||||
@@ -1959,15 +2014,32 @@ Provides: virtual-thing (= 2.0), plain-virtual
|
|||||||
|
|
||||||
// Compare artifacts.
|
// Compare artifacts.
|
||||||
assert_changes_equivalent(
|
assert_changes_equivalent(
|
||||||
&golden_root.join(format!("{NAME}_1.0-1_amd64.changes")),
|
&golden_root.join(format!("{name}_{artifact_version}_amd64.changes")),
|
||||||
&ours_root.join(format!("{NAME}_1.0-1_amd64.changes")),
|
&ours_root.join(format!("{name}_{artifact_version}_amd64.changes")),
|
||||||
);
|
);
|
||||||
assert_buildinfo_equivalent(
|
assert_buildinfo_equivalent(
|
||||||
&golden_root.join(format!("{NAME}_1.0-1_amd64.buildinfo")),
|
&golden_root.join(format!("{name}_{artifact_version}_amd64.buildinfo")),
|
||||||
&ours_root.join(format!("{NAME}_1.0-1_amd64.buildinfo")),
|
&ours_root.join(format!("{name}_{artifact_version}_amd64.buildinfo")),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Differential check of the binary-only (binNMU) metadata against real
|
||||||
|
/// `dpkg-buildpackage -b`: with the previous source artifacts sitting
|
||||||
|
/// next to the tree (as after a source build), the `.changes` must
|
||||||
|
/// distribute only the binaries and the `.buildinfo`, both documents
|
||||||
|
/// referencing the previous version textually only. Regression guard for
|
||||||
|
/// the previous-source redistribution pkh used to emit.
|
||||||
|
#[test]
|
||||||
|
fn diff_binmu_binary_metadata() {
|
||||||
|
const NAME: &str = "pkh-diff-r";
|
||||||
|
let changelog = format!(
|
||||||
|
"{NAME} (1.0-1+b1) unstable; urgency=medium, binary-only=yes\n\n * Binary-only \
|
||||||
|
rebuild.\n\n -- {MAINTAINER} {DATE}\n\n{NAME} (1.0-1) unstable; urgency=medium\n\n \
|
||||||
|
* Initial release.\n\n -- {MAINTAINER} {DATE}\n"
|
||||||
|
);
|
||||||
|
diff_binary_metadata_case(NAME, &changelog, "1.0-1+b1", true);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn diff_native_minimal() {
|
fn diff_native_minimal() {
|
||||||
differential_case(&FixtureSpec::new("pkh-diff-a", "1.0-1", "unstable"));
|
differential_case(&FixtureSpec::new("pkh-diff-a", "1.0-1", "unstable"));
|
||||||
@@ -2039,8 +2111,10 @@ Provides: virtual-thing (= 2.0), plain-virtual
|
|||||||
spec.body = &["* Binary-only rebuild."];
|
spec.body = &["* Binary-only rebuild."];
|
||||||
spec.previous_version = Some("1.0-1");
|
spec.previous_version = Some("1.0-1");
|
||||||
|
|
||||||
// Binary-only metadata references the previous version's .dsc, which
|
// A binNMU-style version number (+b1) with a previous entry, but
|
||||||
// must already exist next to the package tree.
|
// WITHOUT the binary-only marker: this is a plain source build, and
|
||||||
|
// the sibling previous-version .dsc (as left by an earlier source
|
||||||
|
// build) must not change either side's output.
|
||||||
let base = tempfile::tempdir().expect("tempdir");
|
let base = tempfile::tempdir().expect("tempdir");
|
||||||
let tree = write_fixture(base.path(), &spec);
|
let tree = write_fixture(base.path(), &spec);
|
||||||
let prev_dsc = format!(
|
let prev_dsc = format!(
|
||||||
@@ -2055,6 +2129,42 @@ Provides: virtual-thing (= 2.0), plain-virtual
|
|||||||
differential_on_tree(&tree, &SourceBuildOptions::default());
|
differential_on_tree(&tree, &SourceBuildOptions::default());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Both implementations must refuse a source build of a changelog entry
|
||||||
|
/// marked `binary-only=yes`: dpkg-source errors out, and pkh must refuse
|
||||||
|
/// the same way instead of producing binNMU-style source metadata
|
||||||
|
/// referencing the previous version.
|
||||||
|
#[test]
|
||||||
|
fn diff_source_build_rejects_binary_only_marker() {
|
||||||
|
let mut spec = FixtureSpec::new("pkh-diff-s", "1.0-1+b1", "unstable");
|
||||||
|
spec.body = &["* Binary-only rebuild."];
|
||||||
|
spec.previous_version = Some("1.0-1");
|
||||||
|
spec.binary_only_marker = true;
|
||||||
|
let base = tempfile::tempdir().expect("tempdir");
|
||||||
|
let tree = write_fixture(base.path(), &spec);
|
||||||
|
|
||||||
|
// Golden side: real dpkg refuses.
|
||||||
|
let status = crate::test_support::run_logged(
|
||||||
|
Command::new("dpkg-buildpackage").current_dir(&tree).args([
|
||||||
|
"-S",
|
||||||
|
"-I",
|
||||||
|
"-i",
|
||||||
|
"-nc",
|
||||||
|
"-d",
|
||||||
|
"--no-sign",
|
||||||
|
]),
|
||||||
|
)
|
||||||
|
.expect("run dpkg-buildpackage (is dpkg-dev installed?)");
|
||||||
|
assert!(
|
||||||
|
!status.success(),
|
||||||
|
"dpkg-buildpackage -S must refuse a binary-only changelog entry"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Ours: the native pipeline refuses likewise.
|
||||||
|
let err = run_source_build(&tree, &SourceBuildOptions::default(), None)
|
||||||
|
.expect_err("native source build must refuse a binary-only entry");
|
||||||
|
assert!(err.to_string().contains("binary-only"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn diff_closes_bugs() {
|
fn diff_closes_bugs() {
|
||||||
let mut spec = FixtureSpec::new("pkh-diff-j", "2.0-1", "unstable");
|
let mut spec = FixtureSpec::new("pkh-diff-j", "2.0-1", "unstable");
|
||||||
|
|||||||
+260
-42
@@ -31,6 +31,20 @@ const LP_USER_KEY: &str = "lp.user";
|
|||||||
/// Base URL of the Launchpad REST API
|
/// Base URL of the Launchpad REST API
|
||||||
const API_BASE: &str = "https://api.launchpad.net/1.0";
|
const API_BASE: &str = "https://api.launchpad.net/1.0";
|
||||||
|
|
||||||
|
/// Page size (`ws.size`) asked from Launchpad collections. Launchpad
|
||||||
|
/// truncates collection answers at 75 entries by default and rejects
|
||||||
|
/// `ws.size` above 300 (both verified against the live API); 100 sits
|
||||||
|
/// comfortably under the cap while keeping multi-page walks rare.
|
||||||
|
const WS_PAGE_SIZE: u32 = 100;
|
||||||
|
|
||||||
|
/// Hard cap on the pages followed while walking a `getPublishedSources`
|
||||||
|
/// collection: 20 pages x 100 entries = 2000 currently published entries
|
||||||
|
/// for one source name. The query only counts `Published` entries of live
|
||||||
|
/// series/pockets, so real histories are a handful of entries; a walk
|
||||||
|
/// reaching the cap means the API is misbehaving (an endless next-link
|
||||||
|
/// chain), not that the history is genuinely huge.
|
||||||
|
const MAX_COLLECTION_PAGES: u32 = 20;
|
||||||
|
|
||||||
/// The Launchpad username configured in git: the repository-local
|
/// The Launchpad username configured in git: the repository-local
|
||||||
/// configuration wins over the global one, like git's own precedence.
|
/// configuration wins over the global one, like git's own precedence.
|
||||||
/// `None` when no git repository is found or the key is unset.
|
/// `None` when no git repository is found or the key is unset.
|
||||||
@@ -178,20 +192,21 @@ fn percent_encode(value: &str) -> String {
|
|||||||
encoded
|
encoded
|
||||||
}
|
}
|
||||||
|
|
||||||
/// URL of the `getPublishedSources` API call listing the currently
|
/// URL of the first page of the `getPublishedSources` API call listing the
|
||||||
/// `Published` source packages named `source_name` in the PPA `user/name`:
|
/// currently `Published` source packages named `source_name` in the PPA
|
||||||
/// `exact_match` avoids Launchpad's default case-insensitive substring
|
/// `user/name`: `exact_match` avoids Launchpad's default case-insensitive
|
||||||
/// matching, which would return unrelated sources (`data` matching
|
/// substring matching, which would return unrelated sources (`data` matching
|
||||||
/// `datatables`).
|
/// `datatables`). Further pages are reached through the answer's
|
||||||
|
/// `next_collection_link`, not by hand-building URLs.
|
||||||
fn published_sources_url(user: &str, ppa: &str, source_name: &str) -> String {
|
fn published_sources_url(user: &str, ppa: &str, source_name: &str) -> String {
|
||||||
format!(
|
format!(
|
||||||
"{}?ws.op=getPublishedSources&source_name={}&exact_match=true&status=Published",
|
"{}?ws.op=getPublishedSources&source_name={}&exact_match=true&status=Published&ws.size={WS_PAGE_SIZE}",
|
||||||
archive_url(user, ppa),
|
archive_url(user, ppa),
|
||||||
percent_encode(source_name)
|
percent_encode(source_name)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// One entry of a `getPublishedSources` answer: the subset of the source
|
/// One page of a `getPublishedSources` answer: the subset of the source
|
||||||
/// package publishing history the superseded-upload check needs (the live
|
/// package publishing history the superseded-upload check needs (the live
|
||||||
/// answer carries many more fields, ignored by serde)
|
/// answer carries many more fields, ignored by serde)
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
@@ -200,42 +215,52 @@ struct PublishedSource {
|
|||||||
source_package_version: String,
|
source_package_version: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The `getPublishedSources` collection answer
|
/// One page of the `getPublishedSources` collection answer
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
struct PublishedSources {
|
struct PublishedSources {
|
||||||
/// The currently published source packages matching the query
|
/// The currently published source packages matching the query, on this
|
||||||
|
/// page only
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
entries: Vec<PublishedSource>,
|
entries: Vec<PublishedSource>,
|
||||||
|
/// URL of the next page, present only when the collection was
|
||||||
|
/// truncated (Launchpad answers carry it as a plain JSON string)
|
||||||
|
next_collection_link: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Every version of `source_name` currently `Published` in the PPA
|
/// Parse one page of a `getPublishedSources` collection into the versions
|
||||||
/// `user/name` (same `user/ppa_name` format as `pkh put --ppa`), in API
|
/// it carries plus the link to the next page (`None` on the last one): the
|
||||||
/// order. Empty when the source was never published there — a 200 answer
|
/// pagination decision, factored out of the HTTP walk so it can be tested
|
||||||
/// with zero entries, the normal first-upload case.
|
/// without a server.
|
||||||
pub async fn published_versions(
|
fn parse_collection_page(body: &str) -> Result<(Vec<String>, Option<String>), serde_json::Error> {
|
||||||
ppa: &str,
|
let sources: PublishedSources = serde_json::from_str(body)?;
|
||||||
source_name: &str,
|
Ok((
|
||||||
) -> Result<Vec<String>, Box<dyn Error>> {
|
sources
|
||||||
let (user, name) = split_ppa(ppa)?;
|
.entries
|
||||||
let client = crate::distro_info::http_client();
|
.into_iter()
|
||||||
|
.map(|entry| entry.source_package_version)
|
||||||
|
.collect(),
|
||||||
|
sources.next_collection_link,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// GET one page of a collection, mapping the API statuses to the same
|
||||||
|
/// errors as the other Launchpad calls (404 means the PPA does not exist).
|
||||||
|
/// Returns the response body for [`parse_collection_page`].
|
||||||
|
async fn fetch_collection_page(
|
||||||
|
client: &reqwest::Client,
|
||||||
|
url: &str,
|
||||||
|
ppa: &str,
|
||||||
|
) -> Result<String, Box<dyn Error>> {
|
||||||
let response = client
|
let response = client
|
||||||
.get(published_sources_url(&user, &name, source_name))
|
.get(url)
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("cannot reach the Launchpad API: {e}"))?;
|
.map_err(|e| format!("cannot reach the Launchpad API: {e}"))?;
|
||||||
match response.status() {
|
match response.status() {
|
||||||
reqwest::StatusCode::OK => {
|
reqwest::StatusCode::OK => response
|
||||||
let sources: PublishedSources = response
|
.text()
|
||||||
.json()
|
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("cannot parse the Launchpad API response for '{ppa}': {e}"))?;
|
.map_err(|e| format!("cannot read the Launchpad API response for '{ppa}': {e}").into()),
|
||||||
Ok(sources
|
|
||||||
.entries
|
|
||||||
.into_iter()
|
|
||||||
.map(|entry| entry.source_package_version)
|
|
||||||
.collect())
|
|
||||||
}
|
|
||||||
reqwest::StatusCode::NOT_FOUND => {
|
reqwest::StatusCode::NOT_FOUND => {
|
||||||
Err(format!("PPA '{ppa}' does not exist: create it on launchpad.net first").into())
|
Err(format!("PPA '{ppa}' does not exist: create it on launchpad.net first").into())
|
||||||
}
|
}
|
||||||
@@ -243,6 +268,66 @@ pub async fn published_versions(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Walk a `getPublishedSources` collection page by page: fetch the first
|
||||||
|
/// page, then follow `next_collection_link` (the canonical Launchpad
|
||||||
|
/// pagination) until a page comes without one, accumulating the versions of
|
||||||
|
/// every page in order.
|
||||||
|
///
|
||||||
|
/// Exceeding [`MAX_COLLECTION_PAGES`] errors rather than returning the
|
||||||
|
/// partial list: the result feeds `put`'s superseded-upload check, where a
|
||||||
|
/// silently truncated list is exactly the bug pagination fixes — a
|
||||||
|
/// superseded upload wrongly allowed through, to be rejected (or to
|
||||||
|
/// silently supersede) in Launchpad's queue hours later. Every other
|
||||||
|
/// failure mode of this check (network, HTTP status, parsing) aborts the
|
||||||
|
/// upload too, and `put` fails before anything is written, so erring costs
|
||||||
|
/// only a clear message.
|
||||||
|
async fn walk_collection(
|
||||||
|
client: &reqwest::Client,
|
||||||
|
first_url: &str,
|
||||||
|
ppa: &str,
|
||||||
|
) -> Result<Vec<String>, Box<dyn Error>> {
|
||||||
|
let mut versions = Vec::new();
|
||||||
|
let mut url = first_url.to_string();
|
||||||
|
for _page in 1..=MAX_COLLECTION_PAGES {
|
||||||
|
let body = fetch_collection_page(client, &url, ppa).await?;
|
||||||
|
let (mut page_versions, next) = parse_collection_page(&body)
|
||||||
|
.map_err(|e| format!("cannot parse the Launchpad API response for '{ppa}': {e}"))?;
|
||||||
|
versions.append(&mut page_versions);
|
||||||
|
match next {
|
||||||
|
Some(next) => url = next,
|
||||||
|
None => return Ok(versions),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(format!(
|
||||||
|
"the Launchpad API keeps paginating the published sources of '{ppa}' \
|
||||||
|
after {MAX_COLLECTION_PAGES} pages: cannot run the superseded check \
|
||||||
|
on a partial list"
|
||||||
|
)
|
||||||
|
.into())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every version of `source_name` currently `Published` in the PPA
|
||||||
|
/// `user/name` (same `user/ppa_name` format as `pkh put --ppa`), in API
|
||||||
|
/// order. Empty when the source was never published there — a 200 answer
|
||||||
|
/// with zero entries, the normal first-upload case. Launchpad truncates
|
||||||
|
/// collections per page, so the walk follows the API's `next_collection_link`
|
||||||
|
/// until the collection is exhausted: a single page would miss the highest
|
||||||
|
/// version of a source published in many series/pockets over time, and the
|
||||||
|
/// superseded check would wrongly pass.
|
||||||
|
pub async fn published_versions(
|
||||||
|
ppa: &str,
|
||||||
|
source_name: &str,
|
||||||
|
) -> Result<Vec<String>, Box<dyn Error>> {
|
||||||
|
let (user, name) = split_ppa(ppa)?;
|
||||||
|
walk_collection(
|
||||||
|
crate::distro_info::http_client(),
|
||||||
|
&published_sources_url(&user, &name, source_name),
|
||||||
|
ppa,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
/// PPA uploads only target Ubuntu series: fail before uploading when the
|
/// PPA uploads only target Ubuntu series: fail before uploading when the
|
||||||
/// changes' distribution is not a known series (typo) or a non-Ubuntu one —
|
/// changes' distribution is not a known series (typo) or a non-Ubuntu one —
|
||||||
/// both are only rejected during queue processing otherwise
|
/// both are only rejected during queue processing otherwise
|
||||||
@@ -372,16 +457,17 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// The query matches the verified live `getPublishedSources` call, with
|
/// The query matches the verified live `getPublishedSources` call, with
|
||||||
/// the source name percent-encoded
|
/// the source name percent-encoded and an explicit page size (the API
|
||||||
|
/// default of 75 entries would hide part of long publishing histories)
|
||||||
#[test]
|
#[test]
|
||||||
fn published_sources_url_matches_launchpad_call() {
|
fn published_sources_url_matches_launchpad_call() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
published_sources_url("vhaudiquet", "noctalia", "noctalia"),
|
published_sources_url("vhaudiquet", "noctalia", "noctalia"),
|
||||||
"https://api.launchpad.net/1.0/~vhaudiquet/+archive/ubuntu/noctalia?ws.op=getPublishedSources&source_name=noctalia&exact_match=true&status=Published"
|
"https://api.launchpad.net/1.0/~vhaudiquet/+archive/ubuntu/noctalia?ws.op=getPublishedSources&source_name=noctalia&exact_match=true&status=Published&ws.size=100"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
published_sources_url("vhaudiquet", "noctalia", "g++"),
|
published_sources_url("vhaudiquet", "noctalia", "g++"),
|
||||||
"https://api.launchpad.net/1.0/~vhaudiquet/+archive/ubuntu/noctalia?ws.op=getPublishedSources&source_name=g%2B%2B&exact_match=true&status=Published"
|
"https://api.launchpad.net/1.0/~vhaudiquet/+archive/ubuntu/noctalia?ws.op=getPublishedSources&source_name=g%2B%2B&exact_match=true&status=Published&ws.size=100"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -411,21 +497,153 @@ mod tests {
|
|||||||
]
|
]
|
||||||
}"#;
|
}"#;
|
||||||
|
|
||||||
let sources: PublishedSources = serde_json::from_str(json).unwrap();
|
let (versions, next) = parse_collection_page(json).unwrap();
|
||||||
let versions: Vec<&str> = sources
|
|
||||||
.entries
|
|
||||||
.iter()
|
|
||||||
.map(|entry| entry.source_package_version.as_str())
|
|
||||||
.collect();
|
|
||||||
assert_eq!(versions, vec!["5.1.0-1ubuntu2", "2:1.0-1"]);
|
assert_eq!(versions, vec!["5.1.0-1ubuntu2", "2:1.0-1"]);
|
||||||
|
// A page without a next link is the end of the collection
|
||||||
|
assert_eq!(next, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A 200 answer with zero entries is the normal "nothing published
|
/// A 200 answer with zero entries is the normal "nothing published
|
||||||
/// there" case, and must deserialize to an empty list
|
/// there" case, and must deserialize to an empty list
|
||||||
#[test]
|
#[test]
|
||||||
fn published_sources_parses_empty_collection() {
|
fn published_sources_parses_empty_collection() {
|
||||||
let sources: PublishedSources =
|
let (versions, next) =
|
||||||
serde_json::from_str(r#"{"start": 0, "total_size": 0, "entries": []}"#).unwrap();
|
parse_collection_page(r#"{"start": 0, "total_size": 0, "entries": []}"#).unwrap();
|
||||||
assert!(sources.entries.is_empty());
|
assert!(versions.is_empty());
|
||||||
|
assert_eq!(next, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A truncated page announces the next one through
|
||||||
|
/// `next_collection_link`, carried as a plain JSON string (shape
|
||||||
|
/// verified against the live API)
|
||||||
|
#[test]
|
||||||
|
fn parse_collection_page_reads_next_link() {
|
||||||
|
let json = r#"{
|
||||||
|
"start": 0,
|
||||||
|
"total_size": 150,
|
||||||
|
"entries": [{"source_package_version": "1.0-1"}],
|
||||||
|
"next_collection_link": "https://api.launchpad.net/1.0/~u/+archive/ubuntu/p?ws.op=getPublishedSources&ws.size=100&memo=100&ws.start=100"
|
||||||
|
}"#;
|
||||||
|
|
||||||
|
let (versions, next) = parse_collection_page(json).unwrap();
|
||||||
|
assert_eq!(versions, vec!["1.0-1"]);
|
||||||
|
assert_eq!(
|
||||||
|
next.as_deref(),
|
||||||
|
Some(
|
||||||
|
"https://api.launchpad.net/1.0/~u/+archive/ubuntu/p?ws.op=getPublishedSources&ws.size=100&memo=100&ws.start=100"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Serve canned byte responses on a local port, one per connection (the
|
||||||
|
/// last response repeats), and return the listener for URL building
|
||||||
|
///
|
||||||
|
/// The canned responses must use 'Connection: close' so the client opens
|
||||||
|
/// a fresh connection (and receives a fresh response) per request.
|
||||||
|
fn serve_responses(listener: std::net::TcpListener, responses: Vec<String>) {
|
||||||
|
use std::io::{Read, Write};
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
for (served, mut stream) in listener.incoming().flatten().enumerate() {
|
||||||
|
let index = served.min(responses.len() - 1);
|
||||||
|
// Drain the request first: closing with unread inbound data
|
||||||
|
// would send a TCP RST and destroy the response in flight
|
||||||
|
let mut buf = [0u8; 4096];
|
||||||
|
loop {
|
||||||
|
match stream.read(&mut buf) {
|
||||||
|
Ok(0) => break,
|
||||||
|
Ok(n) if buf[..n].windows(4).any(|w| w == b"\r\n\r\n") => break,
|
||||||
|
Ok(_) => continue,
|
||||||
|
Err(_) => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let body = &responses[index];
|
||||||
|
let _ = stream.write_all(
|
||||||
|
format!(
|
||||||
|
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||||
|
body.len()
|
||||||
|
)
|
||||||
|
.as_bytes(),
|
||||||
|
);
|
||||||
|
let _ = stream.flush();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One `getPublishedSources` page carrying `versions`, with the
|
||||||
|
/// `next_collection_link` of a truncated page when `next` is given
|
||||||
|
fn collection_body(versions: &[&str], next: Option<&str>) -> String {
|
||||||
|
let entries: Vec<String> = versions
|
||||||
|
.iter()
|
||||||
|
.map(|v| format!(r#"{{"source_package_version": "{v}"}}"#))
|
||||||
|
.collect();
|
||||||
|
let next_field = next
|
||||||
|
.map(|link| format!(r#", "next_collection_link": "{link}""#))
|
||||||
|
.unwrap_or_default();
|
||||||
|
format!(
|
||||||
|
r#"{{"start": 0, "total_size": {}, "entries": [{}]{next_field}}}"#,
|
||||||
|
versions.len(),
|
||||||
|
entries.join(", ")
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bind a fresh mock server ready to serve `responses` (the caller
|
||||||
|
/// needs the address to build self-referential `next_collection_link`s
|
||||||
|
/// before serving starts)
|
||||||
|
fn bound_collection_server() -> (std::net::TcpListener, String) {
|
||||||
|
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||||
|
(listener, base)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The collection walk follows `next_collection_link`: the versions of
|
||||||
|
/// every page are collected in order, and the walk stops on the page
|
||||||
|
/// without a next link (the mock repeats its last response forever, so
|
||||||
|
/// an extra fetch would still pass — but a missing next-link handling
|
||||||
|
/// would drop page two's versions from the result)
|
||||||
|
#[tokio::test]
|
||||||
|
async fn walk_collection_collects_every_page() {
|
||||||
|
let (listener, base) = bound_collection_server();
|
||||||
|
serve_responses(
|
||||||
|
listener,
|
||||||
|
vec![
|
||||||
|
collection_body(&["1.0-1", "1.6-1"], Some(&format!("{base}/next"))),
|
||||||
|
collection_body(&["0.9-1"], None),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
let versions = walk_collection(
|
||||||
|
crate::distro_info::http_client(),
|
||||||
|
&format!("{base}/~u/+archive/ubuntu/p?ws.op=getPublishedSources"),
|
||||||
|
"u/p",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(versions, vec!["1.0-1", "1.6-1", "0.9-1"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A next-link chain that never ends must error, not loop forever: the
|
||||||
|
/// partial list would feed the superseded check a false "not superseded"
|
||||||
|
#[tokio::test]
|
||||||
|
async fn walk_collection_errors_when_pagination_never_ends() {
|
||||||
|
// The last (only) response repeats forever, each page linking back
|
||||||
|
// to the server: the walk must stop at the page cap by itself
|
||||||
|
let (listener, base) = bound_collection_server();
|
||||||
|
serve_responses(
|
||||||
|
listener,
|
||||||
|
vec![collection_body(&["1.0-1"], Some(&format!("{base}/loop")))],
|
||||||
|
);
|
||||||
|
|
||||||
|
let err = walk_collection(
|
||||||
|
crate::distro_info::http_client(),
|
||||||
|
&format!("{base}/~u/+archive/ubuntu/p?ws.op=getPublishedSources"),
|
||||||
|
"u/p",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err()
|
||||||
|
.to_string();
|
||||||
|
assert!(
|
||||||
|
err.contains("keeps paginating the published sources of 'u/p'"),
|
||||||
|
"unexpected: {err}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -282,11 +282,9 @@ pub fn read_pyproject(path: &Path) -> Option<PyProject> {
|
|||||||
("project", "description") => project.description = Some(value),
|
("project", "description") => project.description = Some(value),
|
||||||
("project", "license") => project.license = license_text(&value),
|
("project", "license") => project.license = license_text(&value),
|
||||||
("project.urls", "Homepage") => project.homepage = Some(value),
|
("project.urls", "Homepage") => project.homepage = Some(value),
|
||||||
("project.scripts", key) => {
|
("project.scripts", key) if project.script.is_none() => {
|
||||||
if project.script.is_none() {
|
|
||||||
project.script = Some(key.to_string());
|
project.script = Some(key.to_string());
|
||||||
}
|
}
|
||||||
}
|
|
||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+359
-16
@@ -6,7 +6,10 @@
|
|||||||
//!
|
//!
|
||||||
//! Payload files are uploaded first and the `.changes` file last, like
|
//! Payload files are uploaded first and the `.changes` file last, like
|
||||||
//! dput does, so a partially uploaded set cannot be picked up by the
|
//! dput does, so a partially uploaded set cannot be picked up by the
|
||||||
//! server-side queue processors.
|
//! server-side queue processors. A run that fails mid-upload removes its
|
||||||
|
//! already-uploaded files from the incoming queue (best effort), so a
|
||||||
|
//! retried upload starts from a clean queue; a failed upload is never
|
||||||
|
//! recorded in the upload log, so a re-run replays every file.
|
||||||
|
|
||||||
pub mod changes;
|
pub mod changes;
|
||||||
pub mod ssh;
|
pub mod ssh;
|
||||||
@@ -96,7 +99,15 @@ pub async fn put(
|
|||||||
// Pre-flight checks for everything the upload queue only rejects after
|
// Pre-flight checks for everything the upload queue only rejects after
|
||||||
// processing: a valid Section, a known target series, and the target
|
// processing: a valid Section, a known target series, and the target
|
||||||
// PPA actually existing (the SFTP queue itself is a blind write)
|
// PPA actually existing (the SFTP queue itself is a blind write)
|
||||||
check_control_section(&opts.cwd, "ubuntu")?;
|
match section_check_target(opts.changes.as_deref(), &opts.cwd) {
|
||||||
|
SectionCheckTarget::Dir(dir) => check_control_section(&dir, "ubuntu")?,
|
||||||
|
SectionCheckTarget::Skip => log::warn!(
|
||||||
|
"cannot check the Section: '{}' has no debian/control next to \
|
||||||
|
it, skipping the pre-flight (the archive still rejects uploads \
|
||||||
|
with unknown sections)",
|
||||||
|
changes_path.display()
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
let checking = multi.add(ProgressBar::new(0));
|
let checking = multi.add(ProgressBar::new(0));
|
||||||
checking.set_style(ui::spinner_style());
|
checking.set_style(ui::spinner_style());
|
||||||
@@ -147,23 +158,50 @@ pub async fn put(
|
|||||||
.to_string();
|
.to_string();
|
||||||
uploads.push((changes_path.clone(), changes_name.clone()));
|
uploads.push((changes_path.clone(), changes_name.clone()));
|
||||||
|
|
||||||
|
let incoming = target.incoming.trim_end_matches('/');
|
||||||
|
|
||||||
|
// Remote names of the files uploaded so far, in upload order (the
|
||||||
|
// .changes last). A run failing mid-upload removes these from the
|
||||||
|
// write-only incoming queue before returning: the uploaded payloads
|
||||||
|
// would otherwise linger in the queue area forever, and a .changes
|
||||||
|
// truncated by a failed close could even be picked up by the scanner.
|
||||||
|
let mut uploaded: Vec<String> = Vec::new();
|
||||||
|
|
||||||
for (path, name) in &uploads {
|
for (path, name) in &uploads {
|
||||||
let size = path
|
let size = match path.metadata() {
|
||||||
.metadata()
|
Ok(metadata) => metadata.len(),
|
||||||
.map_err(|e| format!("cannot stat '{}': {}", path.display(), e))?
|
Err(e) => {
|
||||||
.len();
|
// Nothing was attempted for this file: only what earlier
|
||||||
|
// iterations uploaded needs removing
|
||||||
|
let error: Box<dyn std::error::Error> =
|
||||||
|
format!("cannot stat '{}': {}", path.display(), e).into();
|
||||||
|
cleanup_partial_upload(&sftp, incoming, &uploaded, None, &host);
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
// Same transfer view as pull: prefix line, bar on its own line
|
// Same transfer view as pull: prefix line, bar on its own line
|
||||||
let bar = multi.add(ProgressBar::new(size));
|
let bar = multi.add(ProgressBar::new(size));
|
||||||
bar.enable_steady_tick(std::time::Duration::from_millis(50));
|
bar.enable_steady_tick(std::time::Duration::from_millis(50));
|
||||||
bar.set_style(ui::transfer_style());
|
bar.set_style(ui::transfer_style());
|
||||||
bar.set_prefix(format!("Uploading {name}..."));
|
bar.set_prefix(format!("Uploading {name}..."));
|
||||||
|
|
||||||
let remote = format!("{}/{}", target.incoming.trim_end_matches('/'), name);
|
let remote = format!("{incoming}/{name}");
|
||||||
let result = ssh::upload_file(&sftp, path, &remote, &bar);
|
let result = ssh::upload_file(&sftp, path, &remote, &host, &bar);
|
||||||
bar.finish_and_clear();
|
bar.finish_and_clear();
|
||||||
result?;
|
if let Err(e) = result {
|
||||||
|
// The failed file itself joins the cleanup: its remote `create`
|
||||||
|
// may have succeeded before the failure, leaving a partial — or,
|
||||||
|
// on a failed close, a truncated — file behind
|
||||||
|
cleanup_partial_upload(&sftp, incoming, &uploaded, Some(name), &host);
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
uploaded.push(name.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Recorded only once the whole upload succeeded: the log backs the
|
||||||
|
// duplicate-upload guard, and a failed upload must not count as
|
||||||
|
// uploaded (a re-run replays every file — `sftp.create` truncates, so
|
||||||
|
// replaying is safe).
|
||||||
record_upload(&upload_log_path()?, &record)?;
|
record_upload(&upload_log_path()?, &record)?;
|
||||||
|
|
||||||
// The completion lines replace the summary bar; the guard's own clear
|
// The completion lines replace the summary bar; the guard's own clear
|
||||||
@@ -180,6 +218,48 @@ pub async fn put(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The remote names to attempt removing after a failed upload: everything
|
||||||
|
/// already uploaded plus, when set, `failed` (the file whose upload just
|
||||||
|
/// failed — its remote `create` may have succeeded before the failure,
|
||||||
|
/// leaving a partial or truncated file behind) — in reverse upload order,
|
||||||
|
/// so a `.changes` is removed before the payloads it references and the
|
||||||
|
/// queue scanner never observes the payload set shrinking under a
|
||||||
|
/// still-present `.changes`. Pure so the ordering decision is testable
|
||||||
|
/// without a server; the network side is [`cleanup_partial_upload`].
|
||||||
|
fn cleanup_list(uploaded: &[String], failed: Option<&str>) -> Vec<String> {
|
||||||
|
let mut names: Vec<String> = uploaded.to_vec();
|
||||||
|
if let Some(failed) = failed {
|
||||||
|
names.push(failed.to_string());
|
||||||
|
}
|
||||||
|
names.reverse();
|
||||||
|
names
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Best-effort removal of what a failed upload left in the target's
|
||||||
|
/// incoming queue: the already-uploaded payloads, and a `.changes`
|
||||||
|
/// truncated by a failed close, would otherwise linger in the write-only
|
||||||
|
/// area until the queue is manually cleaned. Launchpad re-validates every
|
||||||
|
/// upload, so this is pure hygiene: a removal failure is logged and
|
||||||
|
/// skipped, and the caller returns the original upload error — never a
|
||||||
|
/// cleanup one.
|
||||||
|
fn cleanup_partial_upload(
|
||||||
|
sftp: &ssh2::Sftp,
|
||||||
|
incoming: &str,
|
||||||
|
uploaded: &[String],
|
||||||
|
failed: Option<&str>,
|
||||||
|
host: &str,
|
||||||
|
) {
|
||||||
|
for name in cleanup_list(uploaded, failed) {
|
||||||
|
let remote = format!("{incoming}/{name}");
|
||||||
|
match ssh::remove_file(sftp, &remote, host) {
|
||||||
|
Ok(()) => info!("Removed leftover {remote} from the failed upload"),
|
||||||
|
Err(e) => {
|
||||||
|
log::warn!("Could not remove the leftover {remote} of the failed upload: {e}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Steady tick interval of every bar rendered by a `put` run
|
/// Steady tick interval of every bar rendered by a `put` run
|
||||||
const TICK: std::time::Duration = std::time::Duration::from_millis(50);
|
const TICK: std::time::Duration = std::time::Duration::from_millis(50);
|
||||||
|
|
||||||
@@ -251,6 +331,40 @@ fn check_control_section(cwd: &Path, dist: &str) -> Result<(), Box<dyn std::erro
|
|||||||
.into())
|
.into())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// What the Section pre-flight decided to validate: a directory holding a
|
||||||
|
/// `debian/control` ([`check_control_section`]'s expected layout), or that
|
||||||
|
/// it cannot run where the upload lives and must be skipped.
|
||||||
|
#[derive(Debug, PartialEq, Eq)]
|
||||||
|
enum SectionCheckTarget {
|
||||||
|
/// Validate `debian/control` in this directory
|
||||||
|
Dir(PathBuf),
|
||||||
|
/// No `debian/control` next to the `.changes`: skip the check
|
||||||
|
Skip,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decide which tree the Section pre-flight validates. An explicit
|
||||||
|
/// `--changes` file must be followed by its own directory — it can name a
|
||||||
|
/// different package than the current tree — so the check runs against the
|
||||||
|
/// `.changes`' directory when it directly holds a `debian/control` (the
|
||||||
|
/// `.changes` inside the package root), and is skipped otherwise: pkh
|
||||||
|
/// writes the `.changes` next to the source tree (an artifacts-only
|
||||||
|
/// directory as far as `debian/control` goes), and checking `cwd` there
|
||||||
|
/// would validate whatever package the user happens to be in. Without
|
||||||
|
/// `--changes` the upload comes from the current tree, which keeps getting
|
||||||
|
/// checked exactly as before. Pure decision, factored out so the
|
||||||
|
/// wrong-tree rule is testable with plain tempdirs.
|
||||||
|
fn section_check_target(changes_path: Option<&Path>, cwd: &Path) -> SectionCheckTarget {
|
||||||
|
let Some(changes) = changes_path else {
|
||||||
|
return SectionCheckTarget::Dir(cwd.to_path_buf());
|
||||||
|
};
|
||||||
|
let dir = changes.parent().unwrap_or_else(|| Path::new("."));
|
||||||
|
if dir.join("debian/control").is_file() {
|
||||||
|
SectionCheckTarget::Dir(dir.to_path_buf())
|
||||||
|
} else {
|
||||||
|
SectionCheckTarget::Skip
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The highest version among `published` that parses as a Debian version,
|
/// The highest version among `published` that parses as a Debian version,
|
||||||
/// `None` when the list is empty or nothing in it parses (entries that
|
/// `None` when the list is empty or nothing in it parses (entries that
|
||||||
/// cannot be parsed are skipped rather than failing the check: the API
|
/// cannot be parsed are skipped rather than failing the check: the API
|
||||||
@@ -419,7 +533,7 @@ fn discover_changes(cwd: &Path) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|||||||
/// upload date. Matching on the digest makes `--force` the only way to
|
/// upload date. Matching on the digest makes `--force` the only way to
|
||||||
/// re-upload an identical file, while a rebuilt file (new digest) never
|
/// re-upload an identical file, while a rebuilt file (new digest) never
|
||||||
/// trips the guard.
|
/// trips the guard.
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
struct UploadRecord {
|
struct UploadRecord {
|
||||||
target: String,
|
target: String,
|
||||||
file: String,
|
file: String,
|
||||||
@@ -461,12 +575,52 @@ fn upload_log_path() -> Result<PathBuf, Box<dyn std::error::Error>> {
|
|||||||
Ok(path)
|
Ok(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Load the upload log, empty when the file does not exist yet
|
/// Load the upload log, empty when the file does not exist yet.
|
||||||
|
///
|
||||||
|
/// The log only backs the advisory duplicate-upload guard (Launchpad
|
||||||
|
/// re-verifies every upload), so a corrupt log must not abort the upload:
|
||||||
|
/// like the context configuration in [`crate::context::manager`], the corrupt
|
||||||
|
/// file is first backed up to `<path>.bak` (best effort) so a later
|
||||||
|
/// [`record_upload`] cannot destroy it, then an empty log is used.
|
||||||
fn load_upload_log(path: &Path) -> Vec<UploadRecord> {
|
fn load_upload_log(path: &Path) -> Vec<UploadRecord> {
|
||||||
std::fs::read_to_string(path)
|
let content = match std::fs::read_to_string(path) {
|
||||||
.ok()
|
Ok(content) => content,
|
||||||
.and_then(|content| serde_json::from_str(&content).ok())
|
Err(e) => {
|
||||||
.unwrap_or_default()
|
if e.kind() != std::io::ErrorKind::NotFound {
|
||||||
|
log::error!(
|
||||||
|
"Cannot read upload log {}: {e}; continuing as if nothing had been uploaded",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
match serde_json::from_str(&content) {
|
||||||
|
Ok(entries) => entries,
|
||||||
|
Err(e) => {
|
||||||
|
log::error!(
|
||||||
|
"Upload log {} is corrupt ({e}); ignoring its content, earlier uploads may not be detected as duplicates anymore",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
backup_corrupt_log(path);
|
||||||
|
Vec::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Back up a corrupt upload log (best effort) so a later
|
||||||
|
/// [`record_upload`] cannot silently destroy its content.
|
||||||
|
fn backup_corrupt_log(path: &Path) {
|
||||||
|
let mut os = path.as_os_str().to_os_string();
|
||||||
|
os.push(".bak");
|
||||||
|
let backup_path = PathBuf::from(os);
|
||||||
|
match std::fs::copy(path, &backup_path) {
|
||||||
|
Ok(_) => log::warn!("Corrupt upload log backed up to {}", backup_path.display()),
|
||||||
|
Err(e) => log::warn!(
|
||||||
|
"Could not back up corrupt upload log to {}: {e}",
|
||||||
|
backup_path.display()
|
||||||
|
),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The previous upload of `record` from the log at `log`, if any (same
|
/// The previous upload of `record` from the log at `log`, if any (same
|
||||||
@@ -484,7 +638,16 @@ fn find_previous_upload(
|
|||||||
fn record_upload(log: &Path, record: &UploadRecord) -> Result<(), Box<dyn std::error::Error>> {
|
fn record_upload(log: &Path, record: &UploadRecord) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let mut entries = load_upload_log(log);
|
let mut entries = load_upload_log(log);
|
||||||
entries.push(record.clone());
|
entries.push(record.clone());
|
||||||
std::fs::write(log, serde_json::to_string_pretty(&entries)?)?;
|
// Write to a temporary file next to the log, then rename it over the
|
||||||
|
// log: rename is atomic within a filesystem, so a crash mid-write leaves
|
||||||
|
// the previous log intact instead of a truncated (corrupt) file.
|
||||||
|
// The temporary file is created with default permissions; the log holds
|
||||||
|
// no secrets and lives in the user's data directory, so the previous
|
||||||
|
// mode is not carried over (same choice as `Files::save_atomic`).
|
||||||
|
let tmp = log.with_extension("new");
|
||||||
|
std::fs::write(&tmp, serde_json::to_string_pretty(&entries)?)
|
||||||
|
.map_err(|e| format!("cannot write '{}': {}", tmp.display(), e))?;
|
||||||
|
std::fs::rename(&tmp, log).map_err(|e| format!("cannot install '{}': {}", log.display(), e))?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -578,6 +741,75 @@ mod tests {
|
|||||||
assert!(err.contains("has no Section"), "unexpected: {err}");
|
assert!(err.contains("has no Section"), "unexpected: {err}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// An explicit `--changes` file of a different package must be checked
|
||||||
|
/// in its own directory (A), not in the current tree (B): B's control
|
||||||
|
/// file decided the pre-flight before the fix
|
||||||
|
#[test]
|
||||||
|
fn section_check_follows_explicit_changes_directory() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let artifacts = dir.path().join("artifacts");
|
||||||
|
let elsewhere = dir.path().join("elsewhere");
|
||||||
|
std::fs::create_dir_all(artifacts.join("debian")).unwrap();
|
||||||
|
std::fs::create_dir_all(elsewhere.join("debian")).unwrap();
|
||||||
|
let changes = artifacts.join("hello_1.0-1_source.changes");
|
||||||
|
std::fs::write(&changes, b"changes").unwrap();
|
||||||
|
std::fs::write(
|
||||||
|
artifacts.join("debian/control"),
|
||||||
|
"Source: hello\nSection: utils\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
std::fs::write(
|
||||||
|
elsewhere.join("debian/control"),
|
||||||
|
"Source: other\nSection: unknown\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
section_check_target(Some(&changes), &elsewhere),
|
||||||
|
SectionCheckTarget::Dir(artifacts.clone())
|
||||||
|
);
|
||||||
|
|
||||||
|
// The decided directory carries the valid section, while the tree
|
||||||
|
// the old code checked would have failed the upload
|
||||||
|
match section_check_target(Some(&changes), &elsewhere) {
|
||||||
|
SectionCheckTarget::Dir(d) => check_control_section(&d, "ubuntu").unwrap(),
|
||||||
|
SectionCheckTarget::Skip => panic!("the .changes directory has debian/control"),
|
||||||
|
}
|
||||||
|
assert!(check_control_section(&elsewhere, "ubuntu").is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A `.changes` in a directory without `debian/control` — pkh's own
|
||||||
|
/// layout puts it next to the source tree, and artifacts can be
|
||||||
|
/// collected away from any source — skips the check instead of
|
||||||
|
/// validating whatever tree `cwd` points at
|
||||||
|
#[test]
|
||||||
|
fn section_check_skips_when_changes_dir_has_no_control() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let artifacts = dir.path().join("artifacts");
|
||||||
|
std::fs::create_dir_all(&artifacts).unwrap();
|
||||||
|
let changes = artifacts.join("hello_1.0-1_source.changes");
|
||||||
|
std::fs::write(&changes, b"changes").unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
section_check_target(Some(&changes), dir.path()),
|
||||||
|
SectionCheckTarget::Skip
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Without `--changes` the upload comes from the current tree: the
|
||||||
|
/// decision stays the cwd, byte-identical to the pre-fix behavior
|
||||||
|
#[test]
|
||||||
|
fn section_check_without_changes_validates_cwd() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let pkg = dir.path().join("hello");
|
||||||
|
std::fs::create_dir_all(pkg.join("debian")).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
section_check_target(None, &pkg),
|
||||||
|
SectionCheckTarget::Dir(pkg)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn discover_errors_when_several_candidates() {
|
fn discover_errors_when_several_candidates() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
@@ -632,6 +864,65 @@ mod tests {
|
|||||||
assert!(find_previous_upload(&log, &rebuilt).unwrap().is_none());
|
assert!(find_previous_upload(&log, &rebuilt).unwrap().is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A plain upload record for the log tests
|
||||||
|
fn upload_log_record(sha256: &str) -> UploadRecord {
|
||||||
|
UploadRecord {
|
||||||
|
target: "ppa:user/ppa".to_string(),
|
||||||
|
file: "hello_1.0-1_source.changes".to_string(),
|
||||||
|
sha256: sha256.to_string(),
|
||||||
|
date: "2025-09-16 12:00".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A corrupt log must not be silently reset: it is backed up byte for
|
||||||
|
/// byte, and the guard restarts from an empty log.
|
||||||
|
#[test]
|
||||||
|
fn corrupt_upload_log_is_backed_up_and_reset() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let log = dir.path().join("uploads.json");
|
||||||
|
let corrupt = b"{ \"not\": valid json";
|
||||||
|
std::fs::write(&log, corrupt).unwrap();
|
||||||
|
|
||||||
|
assert!(load_upload_log(&log).is_empty());
|
||||||
|
|
||||||
|
let backup = dir.path().join("uploads.json.bak");
|
||||||
|
assert_eq!(std::fs::read(&backup).unwrap(), corrupt);
|
||||||
|
|
||||||
|
// Recording after a corrupt log starts a fresh, parseable log
|
||||||
|
// without clobbering the backup
|
||||||
|
record_upload(&log, &upload_log_record("abc")).unwrap();
|
||||||
|
assert_eq!(load_upload_log(&log), vec![upload_log_record("abc")]);
|
||||||
|
assert_eq!(std::fs::read(&backup).unwrap(), corrupt);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `record_upload` leaves a log that parses back, with no temporary
|
||||||
|
/// file left behind
|
||||||
|
#[test]
|
||||||
|
fn recorded_upload_parses_back_atomically() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let log = dir.path().join("uploads.json");
|
||||||
|
|
||||||
|
record_upload(&log, &upload_log_record("abc")).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(load_upload_log(&log), vec![upload_log_record("abc")]);
|
||||||
|
assert!(!dir.path().join("uploads.new").exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Recording a second upload appends to the log instead of replacing it
|
||||||
|
#[test]
|
||||||
|
fn record_over_existing_log_preserves_prior_entries() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let log = dir.path().join("uploads.json");
|
||||||
|
|
||||||
|
record_upload(&log, &upload_log_record("abc")).unwrap();
|
||||||
|
record_upload(&log, &upload_log_record("def")).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
load_upload_log(&log),
|
||||||
|
vec![upload_log_record("abc"), upload_log_record("def")]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/// A minimal in-memory changes file for the superseded-version check
|
/// A minimal in-memory changes file for the superseded-version check
|
||||||
fn changes_fixture(version: &str) -> changes::ChangesFile {
|
fn changes_fixture(version: &str) -> changes::ChangesFile {
|
||||||
changes::ChangesFile {
|
changes::ChangesFile {
|
||||||
@@ -756,4 +1047,56 @@ mod tests {
|
|||||||
let changes = changes_fixture("1.7-1");
|
let changes = changes_fixture("1.7-1");
|
||||||
check_not_superseded(&target, &changes, &published, false).unwrap();
|
check_not_superseded(&target, &changes, &published, false).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The upload loop is network-bound and `ssh2::Sftp` cannot be built
|
||||||
|
/// without a live SSH session, so there is no stub seam for it without
|
||||||
|
/// heavy refactoring; the cleanup decision is factored into the pure
|
||||||
|
/// [`cleanup_list`] and tested directly instead.
|
||||||
|
///
|
||||||
|
/// Failure during the final `.changes` upload (the dangerous case: a
|
||||||
|
/// partial or close-truncated `.changes` is scanner-visible): the
|
||||||
|
/// `.changes` is removed first, then the payloads it references, so the
|
||||||
|
/// queue never sees a `.changes` over a shrinking payload set.
|
||||||
|
#[test]
|
||||||
|
fn cleanup_list_removes_changes_first_then_payloads_in_reverse() {
|
||||||
|
let uploaded = vec![
|
||||||
|
"hello_1.0.orig.tar.xz".to_string(),
|
||||||
|
"hello_1.0-1.debian.tar.xz".to_string(),
|
||||||
|
"hello_1.0-1.dsc".to_string(),
|
||||||
|
];
|
||||||
|
assert_eq!(
|
||||||
|
cleanup_list(&uploaded, Some("hello_1.0-1_source.changes")),
|
||||||
|
vec![
|
||||||
|
"hello_1.0-1_source.changes",
|
||||||
|
"hello_1.0-1.dsc",
|
||||||
|
"hello_1.0-1.debian.tar.xz",
|
||||||
|
"hello_1.0.orig.tar.xz",
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A failure during a payload upload: the `.changes` was never
|
||||||
|
/// attempted (it is uploaded last), so it must not appear in the
|
||||||
|
/// cleanup list — only the failed payload (a partial may exist) and the
|
||||||
|
/// earlier payloads.
|
||||||
|
#[test]
|
||||||
|
fn cleanup_list_before_the_changes_covers_the_failed_payload() {
|
||||||
|
let uploaded = vec!["hello_1.0.orig.tar.xz".to_string()];
|
||||||
|
assert_eq!(
|
||||||
|
cleanup_list(&uploaded, Some("hello_1.0-1.debian.tar.xz")),
|
||||||
|
vec!["hello_1.0-1.debian.tar.xz", "hello_1.0.orig.tar.xz"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A stat failure happens before anything was attempted for that file,
|
||||||
|
/// so only what earlier iterations uploaded is removed — and with
|
||||||
|
/// nothing uploaded yet there is nothing to remove at all.
|
||||||
|
#[test]
|
||||||
|
fn cleanup_list_without_failed_file_keeps_uploaded_only() {
|
||||||
|
assert!(cleanup_list(&[], None).is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
cleanup_list(&["hello_1.0.orig.tar.xz".to_string()], None),
|
||||||
|
vec!["hello_1.0.orig.tar.xz"]
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+378
-24
@@ -5,21 +5,27 @@
|
|||||||
//! authentication (every ssh-agent identity first, then configured and
|
//! authentication (every ssh-agent identity first, then configured and
|
||||||
//! default key files) and chunked SFTP upload with progress reporting.
|
//! default key files) and chunked SFTP upload with progress reporting.
|
||||||
//!
|
//!
|
||||||
|
//! Every network phase is time-bounded — the TCP connect, the SSH
|
||||||
|
//! handshake/authentication, and each low-level call of the SFTP data
|
||||||
|
//! transfer (see the `*_TIMEOUT` constants) — so a black-holed or stalled
|
||||||
|
//! server fails the upload instead of hanging it forever.
|
||||||
|
//!
|
||||||
//! This replaces dput-ng's paramiko transport with the `ssh2` (libssh2)
|
//! This replaces dput-ng's paramiko transport with the `ssh2` (libssh2)
|
||||||
//! stack the rest of pkh already uses for build contexts.
|
//! stack the rest of pkh already uses for build contexts.
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
use std::net::TcpStream;
|
use std::net::{SocketAddr, TcpStream, ToSocketAddrs};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use indicatif::ProgressBar;
|
use indicatif::ProgressBar;
|
||||||
use lazy_static::lazy_static;
|
use lazy_static::lazy_static;
|
||||||
use log::debug;
|
use log::debug;
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use ssh2::{CheckResult, HostKeyType, KnownHostFileKind, Session};
|
use ssh2::{CheckResult, HostKeyType, KnownHostFileKind, KnownHosts, Session};
|
||||||
|
|
||||||
use crate::ui::prompt;
|
use crate::ui::prompt;
|
||||||
|
|
||||||
@@ -221,19 +227,103 @@ fn wildmatch(text: &str, pattern: &str) -> bool {
|
|||||||
pattern[p..].iter().all(|&c| c == '*')
|
pattern[p..].iter().all(|&c| c == '*')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Bound on one TCP connection attempt to one resolved address: `connect(2)`
|
||||||
|
/// would otherwise block for minutes (or forever, behind a silent firewall)
|
||||||
|
/// per address. Generous enough for slow links to Launchpad, short enough
|
||||||
|
/// that a dead target fails in seconds.
|
||||||
|
const TCP_CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
|
||||||
|
|
||||||
|
/// Timeout for the blocking libssh2 calls of the connection phase (banner
|
||||||
|
/// exchange and key exchange of the handshake, host key check,
|
||||||
|
/// authentication): when it expires the pending call fails with
|
||||||
|
/// `LIBSSH2_ERROR_TIMEOUT` (surfaced as "timed out") instead of hanging
|
||||||
|
/// forever on a stalled server. See [`connect`] for what this does and does
|
||||||
|
/// not cover.
|
||||||
|
const SSH_API_TIMEOUT: Duration = Duration::from_secs(30);
|
||||||
|
|
||||||
|
/// Timeout for the blocking libssh2 calls of the SFTP data transfer (file
|
||||||
|
/// creation, chunk writes, final close handshake). It applies *per
|
||||||
|
/// low-level libssh2 call*, not to the whole transfer: the clock restarts
|
||||||
|
/// at every API entry, so the wall-clock duration of a transfer is not
|
||||||
|
/// bounded by design — but a stalled or black-holed server fails one call
|
||||||
|
/// after this budget instead of hanging `pkh put` forever.
|
||||||
|
const SSH_TRANSFER_TIMEOUT: Duration = Duration::from_secs(300);
|
||||||
|
|
||||||
|
/// libssh2 timeouts are millisecond counts; the constants above are exact
|
||||||
|
/// multiples of a millisecond, and anything larger saturates instead of
|
||||||
|
/// truncating
|
||||||
|
fn duration_ms(timeout: Duration) -> u32 {
|
||||||
|
u32::try_from(timeout.as_millis()).unwrap_or(u32::MAX)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve `host:port` and open a TCP connection, trying each resolved
|
||||||
|
/// address in order (like `TcpStream::connect` does) with
|
||||||
|
/// [`TCP_CONNECT_TIMEOUT`] per attempt instead of blocking indefinitely.
|
||||||
|
/// Fails with a message naming the target and every per-address error.
|
||||||
|
fn tcp_connect(host: &str, port: u16) -> Result<TcpStream, String> {
|
||||||
|
let addrs: Vec<SocketAddr> = (host, port)
|
||||||
|
.to_socket_addrs()
|
||||||
|
.map_err(|e| format!("cannot resolve {host}:{port}: {e}"))?
|
||||||
|
.collect();
|
||||||
|
if addrs.is_empty() {
|
||||||
|
return Err(format!(
|
||||||
|
"cannot connect to {host}:{port}: '{host}' resolved to no addresses"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut attempts: Vec<(SocketAddr, std::io::Error)> = Vec::new();
|
||||||
|
for addr in addrs {
|
||||||
|
match TcpStream::connect_timeout(&addr, TCP_CONNECT_TIMEOUT) {
|
||||||
|
Ok(stream) => return Ok(stream),
|
||||||
|
Err(e) => attempts.push((addr, e)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(connect_failed_message(host, port, &attempts))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Message for a target none of whose resolved addresses accepted a
|
||||||
|
/// connection within [`TCP_CONNECT_TIMEOUT`]. Pure so tests can assert it
|
||||||
|
/// without any network.
|
||||||
|
fn connect_failed_message(
|
||||||
|
host: &str,
|
||||||
|
port: u16,
|
||||||
|
attempts: &[(SocketAddr, std::io::Error)],
|
||||||
|
) -> String {
|
||||||
|
let details = attempts
|
||||||
|
.iter()
|
||||||
|
.map(|(addr, e)| format!("{addr}: {e}"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("; ");
|
||||||
|
format!(
|
||||||
|
"cannot connect to {host}:{port} (tried {} address(es)): {details}",
|
||||||
|
attempts.len()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// Connect to `host:port`, verify the server host key and authenticate as
|
/// Connect to `host:port`, verify the server host key and authenticate as
|
||||||
/// `login`: every ssh-agent identity first, then the configured and default
|
/// `login`: every ssh-agent identity first, then the configured and default
|
||||||
/// identity files.
|
/// identity files. The TCP connect, handshake and authentication steps are
|
||||||
|
/// time-bounded ([`TCP_CONNECT_TIMEOUT`], [`SSH_API_TIMEOUT`]).
|
||||||
pub fn connect(
|
pub fn connect(
|
||||||
host: &str,
|
host: &str,
|
||||||
port: u16,
|
port: u16,
|
||||||
login: &str,
|
login: &str,
|
||||||
config: &SshConfig,
|
config: &SshConfig,
|
||||||
) -> Result<Session, Box<dyn std::error::Error>> {
|
) -> Result<Session, Box<dyn std::error::Error>> {
|
||||||
let tcp = TcpStream::connect((host, port))
|
let tcp = tcp_connect(host, port)?;
|
||||||
.map_err(|e| format!("cannot connect to {host}:{port}: {e}"))?;
|
|
||||||
|
|
||||||
let mut session = Session::new()?;
|
let mut session = Session::new()?;
|
||||||
|
// In blocking mode (the libssh2 default), a call that would block loops
|
||||||
|
// in `_libssh2_wait_socket` (via the `BLOCK_ADJUST` macros of
|
||||||
|
// session.h in the vendored libssh2-sys sources), which bounds the
|
||||||
|
// underlying poll()/select() wait by this timeout and fails the call
|
||||||
|
// with LIBSSH2_ERROR_TIMEOUT when it expires. Verified against
|
||||||
|
// libssh2-sys 0.3.3: the handshake (session.c), host key check,
|
||||||
|
// authentication, channel and SFTP calls all route through it, so a
|
||||||
|
// stalled server errors out instead of hanging forever. It bounds
|
||||||
|
// *each* libssh2 call, not the whole phase: a server that trickles
|
||||||
|
// bytes often enough keeps resuming every call in time.
|
||||||
|
session.set_timeout(duration_ms(SSH_API_TIMEOUT));
|
||||||
session.set_tcp_stream(tcp);
|
session.set_tcp_stream(tcp);
|
||||||
session
|
session
|
||||||
.handshake()
|
.handshake()
|
||||||
@@ -246,13 +336,19 @@ pub fn connect(
|
|||||||
|
|
||||||
authenticate(&session, host, login, config)?;
|
authenticate(&session, host, login, config)?;
|
||||||
|
|
||||||
|
// Only SFTP open/data calls remain on this session: switch from the
|
||||||
|
// connection-phase budget to the generous per-call transfer one
|
||||||
|
session.set_timeout(duration_ms(SSH_TRANSFER_TIMEOUT));
|
||||||
|
|
||||||
Ok(session)
|
Ok(session)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check the server's host key, in decreasing order of trust:
|
/// Check the server's host key, in decreasing order of trust:
|
||||||
///
|
///
|
||||||
/// 1. matching one of the host's pinned (published, `host_keys.yml`)
|
/// 1. matching one of the host's pinned (published, `host_keys.yml`)
|
||||||
/// fingerprints — accepted silently;
|
/// fingerprints — accepted (with a diagnostic warning when the user's
|
||||||
|
/// known_hosts records a *different* key for the host, so a stale
|
||||||
|
/// entry gets cleaned up; the pin stays authoritative);
|
||||||
/// 2. matching `~/.ssh/known_hosts` — accepted;
|
/// 2. matching `~/.ssh/known_hosts` — accepted;
|
||||||
/// 3. known-and-different — refused loudly (possible man-in-the-middle);
|
/// 3. known-and-different — refused loudly (possible man-in-the-middle);
|
||||||
/// 4. unknown — fingerprint shown, explicit confirmation required, and on
|
/// 4. unknown — fingerprint shown, explicit confirmation required, and on
|
||||||
@@ -268,21 +364,16 @@ fn verify_host_key(
|
|||||||
|
|
||||||
if host_key_is_pinned(host, &fingerprint) {
|
if host_key_is_pinned(host, &fingerprint) {
|
||||||
debug!("{host} host key matches the published fingerprint");
|
debug!("{host} host key matches the published fingerprint");
|
||||||
|
// Purely diagnostic — the pin already decided — but a known_hosts
|
||||||
|
// entry disagreeing with the published key is worth surfacing
|
||||||
|
log_known_hosts_discrepancy(host, port, key);
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut known_hosts = Session::new()?.known_hosts()?;
|
let mut known_hosts = Session::new()?.known_hosts()?;
|
||||||
for file in known_hosts_files() {
|
load_known_hosts(&mut known_hosts, &known_hosts_files());
|
||||||
// Unreadable/unknown-format lines are skipped by libssh2; a missing
|
|
||||||
// file is fine
|
|
||||||
let _ = known_hosts.read_file(&file, KnownHostFileKind::OpenSSH);
|
|
||||||
}
|
|
||||||
|
|
||||||
let check = if port == 22 {
|
let check = check_known_hosts(&known_hosts, host, port, key);
|
||||||
known_hosts.check(host, key)
|
|
||||||
} else {
|
|
||||||
known_hosts.check_port(host, port, key)
|
|
||||||
};
|
|
||||||
|
|
||||||
match check {
|
match check {
|
||||||
CheckResult::Match => Ok(()),
|
CheckResult::Match => Ok(()),
|
||||||
@@ -320,6 +411,76 @@ fn verify_host_key(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Read `files` into `known_hosts`: silently skips missing files (the
|
||||||
|
/// normal first-use case — there is nothing to consult), warns about a
|
||||||
|
/// file that exists but cannot be read or parsed, naming the file and the
|
||||||
|
/// error — the user should know why their maintained entries are not being
|
||||||
|
/// consulted — and skips it. Unreadable/unknown-format *lines* are skipped
|
||||||
|
/// by libssh2 itself. Returns the emitted warnings, so the decision is
|
||||||
|
/// testable without capturing log output.
|
||||||
|
fn load_known_hosts(known_hosts: &mut KnownHosts, files: &[PathBuf]) -> Vec<String> {
|
||||||
|
let mut warnings = Vec::new();
|
||||||
|
for file in files {
|
||||||
|
if !file.exists() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Err(e) = known_hosts.read_file(file, KnownHostFileKind::OpenSSH) {
|
||||||
|
let warning = format!(
|
||||||
|
"cannot read the known hosts file {}: {e} — its entries are ignored",
|
||||||
|
file.display()
|
||||||
|
);
|
||||||
|
log::warn!("{warning}");
|
||||||
|
warnings.push(warning);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
warnings
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Look up `host`'s `key` in the loaded entries, using the `[host]:port`
|
||||||
|
/// spelling for non-standard ports (the known_hosts encoding)
|
||||||
|
fn check_known_hosts(known_hosts: &KnownHosts, host: &str, port: u16, key: &[u8]) -> CheckResult {
|
||||||
|
if port == 22 {
|
||||||
|
known_hosts.check(host, key)
|
||||||
|
} else {
|
||||||
|
known_hosts.check_port(host, port, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What to tell the user when the server's key matched a pinned (published)
|
||||||
|
/// fingerprint while their known_hosts records a different key for the same
|
||||||
|
/// host — a stale entry, or worse an old compromise artifact they should
|
||||||
|
/// clean up; `None` for every other check result, since the pin is
|
||||||
|
/// authoritative and needs no corroboration. Pure so the decision and the
|
||||||
|
/// wording are testable without a server or a filesystem.
|
||||||
|
fn known_hosts_discrepancy(host: &str, check: CheckResult) -> Option<String> {
|
||||||
|
match check {
|
||||||
|
CheckResult::Mismatch => Some(format!(
|
||||||
|
"the host key of {host} matches Launchpad's published \
|
||||||
|
fingerprint, but your known_hosts file(s) record a DIFFERENT \
|
||||||
|
key for it — proceeding on the published fingerprint; consider \
|
||||||
|
removing the stale '{host}' entry from your known_hosts"
|
||||||
|
)),
|
||||||
|
CheckResult::Match | CheckResult::NotFound | CheckResult::Failure => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Diagnostic-only companion to the pinned-fingerprint acceptance: load the
|
||||||
|
/// user's known hosts files and warn when they record a different key for
|
||||||
|
/// `host` than the pinned one about to be accepted. The pin stays
|
||||||
|
/// authoritative: any failure here is tolerated and the connection
|
||||||
|
/// proceeds regardless.
|
||||||
|
fn log_known_hosts_discrepancy(host: &str, port: u16, key: &[u8]) {
|
||||||
|
let Ok(mut known_hosts) = Session::new().and_then(|session| session.known_hosts()) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
load_known_hosts(&mut known_hosts, &known_hosts_files());
|
||||||
|
|
||||||
|
let check = check_known_hosts(&known_hosts, host, port, key);
|
||||||
|
if let Some(warning) = known_hosts_discrepancy(host, check) {
|
||||||
|
log::warn!("{warning}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Known hosts files to consult, user file first (so that new keys are
|
/// Known hosts files to consult, user file first (so that new keys are
|
||||||
/// accepted because of the user file, mirroring ssh's own ordering)
|
/// accepted because of the user file, mirroring ssh's own ordering)
|
||||||
fn known_hosts_files() -> Vec<PathBuf> {
|
fn known_hosts_files() -> Vec<PathBuf> {
|
||||||
@@ -426,14 +587,17 @@ fn authenticate(
|
|||||||
.into())
|
.into())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Upload `local` to the remote SFTP `path`, updating `bar` per chunk. The
|
/// Upload `local` to the remote SFTP `path` on `host`, updating `bar` per
|
||||||
/// remote file is created/truncated; Launchpad's upload queue is
|
/// chunk. The remote file is created/truncated; Launchpad's upload queue is
|
||||||
/// write-only, so failures here mean the upload failed — there is nothing
|
/// write-only, so failures here mean the upload failed — there is nothing
|
||||||
/// to inspect server-side.
|
/// to inspect server-side. Every SFTP call is bounded by the session's
|
||||||
|
/// per-call transfer timeout (see [`SSH_TRANSFER_TIMEOUT`]), so a stalled
|
||||||
|
/// server fails the upload instead of hanging it.
|
||||||
pub fn upload_file(
|
pub fn upload_file(
|
||||||
sftp: &ssh2::Sftp,
|
sftp: &ssh2::Sftp,
|
||||||
local: &Path,
|
local: &Path,
|
||||||
remote: &str,
|
remote: &str,
|
||||||
|
host: &str,
|
||||||
bar: &ProgressBar,
|
bar: &ProgressBar,
|
||||||
) -> Result<(), Box<dyn std::error::Error>> {
|
) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let mut local_file =
|
let mut local_file =
|
||||||
@@ -441,7 +605,7 @@ pub fn upload_file(
|
|||||||
|
|
||||||
let mut remote_file = sftp
|
let mut remote_file = sftp
|
||||||
.create(Path::new(remote))
|
.create(Path::new(remote))
|
||||||
.map_err(|e| format!("cannot create remote file {remote}: {e}"))?;
|
.map_err(|e| format!("cannot create remote file {remote} on {host}: {e}"))?;
|
||||||
|
|
||||||
let mut buf = [0u8; 32 * 1024];
|
let mut buf = [0u8; 32 * 1024];
|
||||||
loop {
|
loop {
|
||||||
@@ -451,7 +615,7 @@ pub fn upload_file(
|
|||||||
}
|
}
|
||||||
remote_file
|
remote_file
|
||||||
.write_all(&buf[..n])
|
.write_all(&buf[..n])
|
||||||
.map_err(|e| format!("failed uploading to {remote}: {e}"))?;
|
.map_err(|e| format!("failed uploading to {remote} on {host}: {e}"))?;
|
||||||
bar.inc(n as u64);
|
bar.inc(n as u64);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -461,13 +625,25 @@ pub fn upload_file(
|
|||||||
// truncated remote file as a successful upload. `ssh2::File::write` is
|
// truncated remote file as a successful upload. `ssh2::File::write` is
|
||||||
// unbuffered (`Write::flush` is a documented no-op) and `close`
|
// unbuffered (`Write::flush` is a documented no-op) and `close`
|
||||||
// finalizes the pending writes server-side, so no flush is needed.
|
// finalizes the pending writes server-side, so no flush is needed.
|
||||||
remote_file
|
remote_file.close().map_err(|e| {
|
||||||
.close()
|
format!("failed to close remote file '{remote}' on {host} after upload: {e}")
|
||||||
.map_err(|e| format!("failed to close remote file '{remote}' after upload: {e}"))?;
|
})?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Remove the remote file `remote` on `host` (SFTP remove). Used by the
|
||||||
|
/// best-effort cleanup of an interrupted upload: the queue is write-only,
|
||||||
|
/// so the only remote operation ever needed besides `create` is this one.
|
||||||
|
pub fn remove_file(
|
||||||
|
sftp: &ssh2::Sftp,
|
||||||
|
remote: &str,
|
||||||
|
host: &str,
|
||||||
|
) -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
sftp.unlink(Path::new(remote))
|
||||||
|
.map_err(|e| format!("cannot remove remote file {remote} on {host}: {e}").into())
|
||||||
|
}
|
||||||
|
|
||||||
/// Open the SFTP subsystem on `session`
|
/// Open the SFTP subsystem on `session`
|
||||||
pub fn sftp(session: &Session) -> Result<ssh2::Sftp, Box<dyn std::error::Error>> {
|
pub fn sftp(session: &Session) -> Result<ssh2::Sftp, Box<dyn std::error::Error>> {
|
||||||
session
|
session
|
||||||
@@ -589,6 +765,99 @@ mod tests {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A pinned-key host whose known_hosts entries record a DIFFERENT key
|
||||||
|
/// is flagged with a message naming the host; every other check result
|
||||||
|
/// stays silent — the published fingerprint remains authoritative.
|
||||||
|
#[test]
|
||||||
|
fn known_hosts_discrepancy_flags_mismatch_only() {
|
||||||
|
let message = known_hosts_discrepancy("ppa.launchpad.net", CheckResult::Mismatch)
|
||||||
|
.expect("a mismatch under a matching pin must be flagged");
|
||||||
|
assert!(
|
||||||
|
message.contains("ppa.launchpad.net"),
|
||||||
|
"unexpected: {message}"
|
||||||
|
);
|
||||||
|
assert!(message.contains("DIFFERENT"), "unexpected: {message}");
|
||||||
|
assert!(
|
||||||
|
message.contains("published fingerprint"),
|
||||||
|
"unexpected: {message}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
message.contains("stale"),
|
||||||
|
"the user must be told to clean the entry up: {message}"
|
||||||
|
);
|
||||||
|
|
||||||
|
for quiet in [
|
||||||
|
CheckResult::Match,
|
||||||
|
CheckResult::NotFound,
|
||||||
|
CheckResult::Failure,
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
known_hosts_discrepancy("ppa.launchpad.net", quiet).is_none(),
|
||||||
|
"{quiet:?} must not be flagged"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Known hosts loading decisions: a missing file is the normal
|
||||||
|
/// first-use case and stays silent, a readable file is loaded and its
|
||||||
|
/// entries consulted, and an existing but unreadable file yields a
|
||||||
|
/// warning naming it — the user must learn why their maintained
|
||||||
|
/// entries are not being consulted. Offline: a bare session suffices
|
||||||
|
/// for the known-hosts store.
|
||||||
|
#[test]
|
||||||
|
fn load_known_hosts_warns_only_on_unreadable_files() {
|
||||||
|
use std::os::unix::fs::PermissionsExt;
|
||||||
|
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let mut known_hosts = Session::new().unwrap().known_hosts().unwrap();
|
||||||
|
|
||||||
|
// A missing file is silent...
|
||||||
|
let missing = dir.path().join("known_hosts");
|
||||||
|
assert!(load_known_hosts(&mut known_hosts, &[missing]).is_empty());
|
||||||
|
|
||||||
|
// ...a readable one is loaded and consulted...
|
||||||
|
let key = b"a test host key";
|
||||||
|
let loaded = dir.path().join("loaded_known_hosts");
|
||||||
|
fs::write(
|
||||||
|
&loaded,
|
||||||
|
format!("host.example.com ssh-ed25519 {}\n", base64_nopad(key)),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
load_known_hosts(&mut known_hosts, &[loaded]).is_empty(),
|
||||||
|
"a readable file must not warn"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
matches!(
|
||||||
|
check_known_hosts(&known_hosts, "host.example.com", 22, key),
|
||||||
|
CheckResult::Match
|
||||||
|
),
|
||||||
|
"the loaded entry must be consulted"
|
||||||
|
);
|
||||||
|
|
||||||
|
// ...and an existing but unreadable one is reported, naming the
|
||||||
|
// file. Environments that read through the mode bits (root) cannot
|
||||||
|
// exercise the unreadable case and are skipped.
|
||||||
|
let locked = dir.path().join("locked_known_hosts");
|
||||||
|
fs::write(&locked, b"stale.example.com ssh-ed25519 c3RhbGU=").unwrap();
|
||||||
|
let mut permissions = fs::metadata(&locked).unwrap().permissions();
|
||||||
|
permissions.set_mode(0o000);
|
||||||
|
fs::set_permissions(&locked, permissions).unwrap();
|
||||||
|
if fs::read(&locked).is_ok() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let warnings = load_known_hosts(&mut known_hosts, &[locked]);
|
||||||
|
assert_eq!(warnings.len(), 1, "unexpected: {warnings:?}");
|
||||||
|
assert!(
|
||||||
|
warnings[0].contains("locked_known_hosts"),
|
||||||
|
"the warning must name the file: {warnings:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
warnings[0].contains("ignored"),
|
||||||
|
"the warning must say the entries are not consulted: {warnings:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/// Negated `Host` patterns exclude the host from the whole block (the
|
/// Negated `Host` patterns exclude the host from the whole block (the
|
||||||
/// OpenSSH rule): a block applies iff at least one positive pattern
|
/// OpenSSH rule): a block applies iff at least one positive pattern
|
||||||
/// matches AND no negated pattern matches.
|
/// matches AND no negated pattern matches.
|
||||||
@@ -761,4 +1030,89 @@ Host other
|
|||||||
apply_config_file(&mut config, content, host);
|
apply_config_file(&mut config, content, host);
|
||||||
config
|
config
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The all-addresses-failed message names the target and reports every
|
||||||
|
/// attempted address with its error. Pure: no network involved.
|
||||||
|
#[test]
|
||||||
|
fn connect_failed_message_names_target_and_attempts() {
|
||||||
|
let attempts = vec![
|
||||||
|
(
|
||||||
|
SocketAddr::new(
|
||||||
|
std::net::IpAddr::V4(std::net::Ipv4Addr::new(192, 0, 2, 10)),
|
||||||
|
22,
|
||||||
|
),
|
||||||
|
std::io::Error::new(std::io::ErrorKind::TimedOut, "connection timed out"),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
SocketAddr::new(std::net::IpAddr::V6(std::net::Ipv6Addr::LOCALHOST), 22),
|
||||||
|
std::io::Error::new(std::io::ErrorKind::ConnectionRefused, "refused"),
|
||||||
|
),
|
||||||
|
];
|
||||||
|
|
||||||
|
let message = connect_failed_message("ppa.launchpad.net", 22, &attempts);
|
||||||
|
assert!(
|
||||||
|
message.contains("cannot connect to ppa.launchpad.net:22"),
|
||||||
|
"unexpected: {message}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
message.contains("tried 2 address(es)"),
|
||||||
|
"unexpected: {message}"
|
||||||
|
);
|
||||||
|
assert!(message.contains("192.0.2.10:22"), "unexpected: {message}");
|
||||||
|
assert!(
|
||||||
|
message.contains("connection timed out"),
|
||||||
|
"unexpected: {message}"
|
||||||
|
);
|
||||||
|
assert!(message.contains("[::1]:22"), "unexpected: {message}");
|
||||||
|
assert!(message.contains("refused"), "unexpected: {message}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The whole `tcp_connect` failure path on a loopback port with no
|
||||||
|
/// listener (numeric literal, so no DNS; loopback only, so no external
|
||||||
|
/// network): the per-address `connect_timeout` loop produces the
|
||||||
|
/// formatted message naming the target. DNS resolution failures are
|
||||||
|
/// deliberately not tested: resolving a bogus name would touch the
|
||||||
|
/// system resolver, which is not offline-safe.
|
||||||
|
#[test]
|
||||||
|
fn tcp_connect_closed_port_fails_naming_the_target() {
|
||||||
|
// Grab a free port, then release it: connecting to the now-closed
|
||||||
|
// port fails immediately (ECONNREFUSED), never reaching the timeout
|
||||||
|
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let addr = listener.local_addr().unwrap();
|
||||||
|
drop(listener);
|
||||||
|
|
||||||
|
let error = tcp_connect("127.0.0.1", addr.port()).unwrap_err();
|
||||||
|
assert!(
|
||||||
|
error.contains(&format!("cannot connect to 127.0.0.1:{}", addr.port())),
|
||||||
|
"unexpected: {error}"
|
||||||
|
);
|
||||||
|
assert!(error.contains("tried 1 address(es)"), "unexpected: {error}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The libssh2 timeout wiring round-trips through the session API with
|
||||||
|
/// the configured constants (no network: a bare session suffices)
|
||||||
|
#[test]
|
||||||
|
fn session_timeouts_use_the_configured_constants() {
|
||||||
|
let session = Session::new().unwrap();
|
||||||
|
|
||||||
|
session.set_timeout(duration_ms(SSH_API_TIMEOUT));
|
||||||
|
assert_eq!(session.timeout(), duration_ms(SSH_API_TIMEOUT));
|
||||||
|
|
||||||
|
// The transfer budget replaces the connection one once authenticated
|
||||||
|
session.set_timeout(duration_ms(SSH_TRANSFER_TIMEOUT));
|
||||||
|
assert_eq!(session.timeout(), duration_ms(SSH_TRANSFER_TIMEOUT));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Millisecond conversion is exact for the constants and saturates for
|
||||||
|
/// larger values instead of truncating or panicking
|
||||||
|
#[test]
|
||||||
|
fn duration_ms_converts_and_saturates() {
|
||||||
|
assert_eq!(duration_ms(TCP_CONNECT_TIMEOUT), 15_000);
|
||||||
|
assert_eq!(duration_ms(SSH_API_TIMEOUT), 30_000);
|
||||||
|
assert_eq!(duration_ms(SSH_TRANSFER_TIMEOUT), 300_000);
|
||||||
|
assert_eq!(
|
||||||
|
duration_ms(Duration::from_millis(u64::from(u32::MAX) + 1)),
|
||||||
|
u32::MAX
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-6
@@ -379,12 +379,10 @@ fn text_inner(
|
|||||||
input.push(c);
|
input.push(c);
|
||||||
render(Render::Line(&input))?;
|
render(Render::Line(&input))?;
|
||||||
}
|
}
|
||||||
event::KeyCode::Backspace => {
|
event::KeyCode::Backspace if !input.is_empty() => {
|
||||||
if !input.is_empty() {
|
|
||||||
input.pop();
|
input.pop();
|
||||||
render(Render::Line(&input))?;
|
render(Render::Line(&input))?;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -433,12 +431,10 @@ fn confirm_inner(
|
|||||||
input.push(c);
|
input.push(c);
|
||||||
render(Render::Line(&input))?;
|
render(Render::Line(&input))?;
|
||||||
}
|
}
|
||||||
event::KeyCode::Backspace => {
|
event::KeyCode::Backspace if !input.is_empty() => {
|
||||||
if !input.is_empty() {
|
|
||||||
input.pop();
|
input.pop();
|
||||||
render(Render::Line(&input))?;
|
render(Render::Line(&input))?;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
Reference in New Issue
Block a user