Handle malformed remote and edge-case data instead of panicking

- distro_info: malformed CSV rows are skipped with a warning, dates
  that fail to parse become None, and all plain HTTP requests go
  through a shared reqwest client with connect/total timeouts
- package_info: the Sources stanza iterator is iterative (a crafted
  index with many blank stanzas overflowed the stack), stanzas missing
  a Version are skipped, and failed series/pocket probes are summarized
  in the final 'not found' error instead of being silently dropped
- pull: no double unwrap on the remote-derived artifact filename, an
  empty series list is an error, and streaming downloads get a
  per-request timeout
- deb/cross: dpkg-architecture output parsing skips unexpected lines
  and its exit status is checked, as is dpkg --add-architecture
- changelog: version increments parse as u64 with checked arithmetic
  (1.0-20250123123456 used to panic on the u32 parse)
This commit is contained in:
2026-09-16 02:44:45 +02:00
parent 6a5c5a7106
commit f72b35acfa
5 changed files with 489 additions and 142 deletions
+68 -23
View File
@@ -40,7 +40,7 @@ pub fn generate_entry(
version.to_string() version.to_string()
} else { } else {
// TODO: Pass these flags from CLI // TODO: Pass these flags from CLI
compute_new_version(&old_version, false, false, false) compute_new_version(&old_version, false, false, false)?
}; };
let (maintainer_name, maintainer_email) = get_maintainer_info()?; let (maintainer_name, maintainer_email) = get_maintainer_info()?;
@@ -68,7 +68,7 @@ fn compute_new_version(
is_ubuntu: bool, is_ubuntu: bool,
is_rebuild: bool, is_rebuild: bool,
is_nmu: bool, is_nmu: bool,
) -> String { ) -> Result<String, Box<dyn std::error::Error>> {
if is_ubuntu { if is_ubuntu {
return increment_suffix(old_version, "ubuntu"); return increment_suffix(old_version, "ubuntu");
} }
@@ -86,7 +86,7 @@ fn compute_new_version(
} }
/// Increment a version number by 1, for a given suffix /// Increment a version number by 1, for a given suffix
fn increment_suffix(version: &str, suffix: &str) -> String { fn increment_suffix(version: &str, suffix: &str) -> Result<String, Box<dyn std::error::Error>> {
// If suffix is empty, we just look for trailing digits // If suffix is empty, we just look for trailing digits
// If suffix is not empty, we look for suffix followed by digits // If suffix is not empty, we look for suffix followed by digits
@@ -100,19 +100,33 @@ fn increment_suffix(version: &str, suffix: &str) -> String {
if let Some(caps) = re.captures(version) { if let Some(caps) = re.captures(version) {
let num_str = caps.get(1).unwrap().as_str(); let num_str = caps.get(1).unwrap().as_str();
let num: u32 = num_str.parse().unwrap(); // Parse as u64 so that large trailing numbers (e.g. date-based
// versions like '1.0-20250123123456', which do not fit in a u32)
// still increment normally
let num: u64 = num_str.parse().map_err(|_| {
format!(
"Cannot increment version '{version}': trailing number '{num_str}' \
is too large to be incremented. Specify a version explicitly instead."
)
})?;
let range = caps.get(1).unwrap().range(); let range = caps.get(1).unwrap().range();
let new_num = num.checked_add(1).ok_or_else(|| {
format!(
"Cannot increment version '{version}': trailing number {num} \
is too large to be incremented. Specify a version explicitly instead."
)
})?;
let mut new_ver = version.to_string(); let mut new_ver = version.to_string();
new_ver.replace_range(range, &(num + 1).to_string()); new_ver.replace_range(range, &new_num.to_string());
return new_ver; return Ok(new_ver);
} }
// If pattern not found, append suffix + "1" // If pattern not found, append suffix + "1"
// But if suffix is empty, we default to appending "-1" (standard Debian revision start) // But if suffix is empty, we default to appending "-1" (standard Debian revision start)
if suffix.is_empty() { if suffix.is_empty() {
format!("{}-1", version) Ok(format!("{}-1", version))
} else { } else {
format!("{}{}{}", version, suffix, 1) Ok(format!("{}{}{}", version, suffix, 1))
} }
} }
@@ -381,70 +395,101 @@ mod tests {
fn test_compute_new_version() { fn test_compute_new_version() {
// Debian upload // Debian upload
assert_eq!( assert_eq!(
compute_new_version("15.2.0-8", false, false, false), compute_new_version("15.2.0-8", false, false, false).unwrap(),
"15.2.0-9" "15.2.0-9"
); );
assert_eq!( assert_eq!(
compute_new_version("15.2.0-9", false, false, false), compute_new_version("15.2.0-9", false, false, false).unwrap(),
"15.2.0-10" "15.2.0-10"
); );
// Ubuntu upload // Ubuntu upload
assert_eq!( assert_eq!(
compute_new_version("15.2.0-9", true, false, false), compute_new_version("15.2.0-9", true, false, false).unwrap(),
"15.2.0-9ubuntu1" "15.2.0-9ubuntu1"
); );
assert_eq!( assert_eq!(
compute_new_version("15.2.0-9ubuntu1", true, false, false), compute_new_version("15.2.0-9ubuntu1", true, false, false).unwrap(),
"15.2.0-9ubuntu2" "15.2.0-9ubuntu2"
); );
// No change rebuild // No change rebuild
assert_eq!( assert_eq!(
compute_new_version("15.2.0-9", false, true, false), compute_new_version("15.2.0-9", false, true, false).unwrap(),
"15.2.0-9build1" "15.2.0-9build1"
); );
assert_eq!( assert_eq!(
compute_new_version("15.2.0-9build1", false, true, false), compute_new_version("15.2.0-9build1", false, true, false).unwrap(),
"15.2.0-9build2" "15.2.0-9build2"
); );
// Rebuild of Ubuntu version // Rebuild of Ubuntu version
assert_eq!( assert_eq!(
compute_new_version("15.2.0-9ubuntu1", false, true, false), compute_new_version("15.2.0-9ubuntu1", false, true, false).unwrap(),
"15.2.0-9ubuntu1build1" "15.2.0-9ubuntu1build1"
); );
// NMU // NMU
// Native // Native
assert_eq!(compute_new_version("1.0", false, false, true), "1.0+nmu1");
assert_eq!( assert_eq!(
compute_new_version("1.0+nmu1", false, false, true), compute_new_version("1.0", false, false, true).unwrap(),
"1.0+nmu1"
);
assert_eq!(
compute_new_version("1.0+nmu1", false, false, true).unwrap(),
"1.0+nmu2" "1.0+nmu2"
); );
// Non-native // Non-native
assert_eq!(compute_new_version("1.0-1", false, false, true), "1.0-1.1");
assert_eq!( assert_eq!(
compute_new_version("1.0-1.1", false, false, true), compute_new_version("1.0-1", false, false, true).unwrap(),
"1.0-1.1"
);
assert_eq!(
compute_new_version("1.0-1.1", false, false, true).unwrap(),
"1.0-1.2" "1.0-1.2"
); );
// NMU of NMU? // NMU of NMU?
assert_eq!( assert_eq!(
compute_new_version("1.0-1.2", false, false, true), compute_new_version("1.0-1.2", false, false, true).unwrap(),
"1.0-1.3" "1.0-1.3"
); );
// Native package uploads // Native package uploads
assert_eq!(compute_new_version("1.0", false, false, false), "1.1");
assert_eq!(compute_new_version("1.0.5", false, false, false), "1.0.6");
assert_eq!( assert_eq!(
compute_new_version("20241126", false, false, false), compute_new_version("1.0", false, false, false).unwrap(),
"1.1"
);
assert_eq!(
compute_new_version("1.0.5", false, false, false).unwrap(),
"1.0.6"
);
assert_eq!(
compute_new_version("20241126", false, false, false).unwrap(),
"20241127" "20241127"
); );
} }
#[test]
fn test_compute_new_version_large_trailing_number() {
// Date-based versions with a trailing number larger than u32::MAX
// must increment normally (they fit in a u64)
assert_eq!(
compute_new_version("1.0-20250123123456", false, false, false).unwrap(),
"1.0-20250123123457"
);
// A number that does not even fit in a u64 yields a clear error
// instead of panicking
let err = compute_new_version("1.0-99999999999999999999999999", false, false, false);
assert!(err.is_err());
// u64::MAX itself cannot be incremented
let err = compute_new_version("1.0-18446744073709551615", false, false, false);
assert!(err.is_err());
}
#[test] #[test]
fn test_get_maintainer_info() { fn test_get_maintainer_info() {
// Test with env vars // Test with env vars
+110 -17
View File
@@ -1,24 +1,19 @@
use crate::context::Context; use crate::context::Context;
use log::debug;
use std::collections::HashMap; use std::collections::HashMap;
use std::error::Error; use std::error::Error;
use std::sync::Arc; use std::sync::Arc;
/// Set environment variables for cross-compilation /// Parse 'dpkg-architecture' output (KEY=value lines) into a set of
pub fn setup_environment( /// environment variables. Unexpected lines (e.g. warnings on stderr leaking
env: &mut HashMap<String, String>, /// into stdout) are skipped instead of causing a failure.
arch: &str, fn parse_dpkg_architecture_output(output: &str, env: &mut HashMap<String, String>) {
ctx: Arc<Context>,
) -> Result<(), Box<dyn Error>> {
let dpkg_architecture = String::from_utf8(
ctx.command("dpkg-architecture")
.arg("-a")
.arg(arch)
.output()?
.stdout,
)?;
let env_var_regex = regex::Regex::new(r"(?<key>.*)=(?<value>.*)").unwrap(); let env_var_regex = regex::Regex::new(r"(?<key>.*)=(?<value>.*)").unwrap();
for l in dpkg_architecture.lines() { for l in output.lines() {
let capture = env_var_regex.captures(l).unwrap(); let Some(capture) = env_var_regex.captures(l) else {
debug!("Skipping unexpected dpkg-architecture output line: '{l}'");
continue;
};
let key = capture.name("key").unwrap().as_str().to_string(); let key = capture.name("key").unwrap().as_str().to_string();
let value = capture.name("value").unwrap().as_str().to_string(); let value = capture.name("value").unwrap().as_str().to_string();
@@ -28,6 +23,45 @@ pub fn setup_environment(
env.insert("CROSS_COMPILE".to_string(), format!("{value}-")); env.insert("CROSS_COMPILE".to_string(), format!("{value}-"));
} }
} }
}
/// Set environment variables for cross-compilation
pub fn setup_environment(
env: &mut HashMap<String, String>,
arch: &str,
ctx: Arc<Context>,
) -> Result<(), Box<dyn Error>> {
let output = ctx
.command("dpkg-architecture")
.arg("-a")
.arg(arch)
.output()
.map_err(|e| {
format!(
"Failed to run 'dpkg-architecture -a {arch}': {e}. \
Is 'dpkg-dev' installed?"
)
})?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(format!(
"'dpkg-architecture -a {}' failed with status: {}.{}",
arch,
output.status,
if stderr.trim().is_empty() {
String::new()
} else {
format!("\ndpkg-architecture output:\n{}", stderr.trim())
}
)
.into());
}
let dpkg_architecture = String::from_utf8(output.stdout)
.map_err(|e| format!("Invalid UTF-8 in dpkg-architecture output: {e}"))?;
parse_dpkg_architecture_output(&dpkg_architecture, env);
env.insert("DEB_BUILD_PROFILES".to_string(), "cross".to_string()); env.insert("DEB_BUILD_PROFILES".to_string(), "cross".to_string());
Ok(()) Ok(())
@@ -44,10 +78,19 @@ pub fn ensure_repositories(
let local_arch = crate::get_current_arch(); let local_arch = crate::get_current_arch();
// Add target ('host') architecture // Add target ('host') architecture
ctx.command("dpkg") let status = ctx
.command("dpkg")
.arg("--add-architecture") .arg("--add-architecture")
.arg(arch) .arg(arch)
.status()?; .status()
.map_err(|e| format!("Failed to run 'dpkg --add-architecture {arch}': {e}"))?;
if !status.success() {
return Err(format!(
"'dpkg --add-architecture {}' failed with status: {}",
arch, status
)
.into());
}
// Check if we are on Ubuntu // Check if we are on Ubuntu
let os_release = String::from_utf8(ctx.command("cat").arg("/etc/os-release").output()?.stdout)?; let os_release = String::from_utf8(ctx.command("cat").arg("/etc/os-release").output()?.stdout)?;
@@ -142,3 +185,53 @@ pub fn ensure_repositories(
Ok(()) Ok(())
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_dpkg_architecture_output() {
let output = "DEB_BUILD_ARCH=amd64\n\
DEB_HOST_ARCH=arm64\n\
DEB_HOST_GNU_TYPE=aarch64-linux-gnu\n";
let mut env = HashMap::new();
parse_dpkg_architecture_output(output, &mut env);
assert_eq!(env.get("DEB_BUILD_ARCH").map(String::as_str), Some("amd64"));
assert_eq!(env.get("DEB_HOST_ARCH").map(String::as_str), Some("arm64"));
assert_eq!(
env.get("DEB_HOST_GNU_TYPE").map(String::as_str),
Some("aarch64-linux-gnu")
);
// Derived variable for the GNU type
assert_eq!(
env.get("CROSS_COMPILE").map(String::as_str),
Some("aarch64-linux-gnu-")
);
}
#[test]
fn test_parse_dpkg_architecture_output_skips_unexpected_lines() {
// Unexpected lines (warnings on stdout, empty lines) must be skipped
// instead of panicking
let output = "dpkg-architecture: warning: something odd happened\n\
\n\
DEB_HOST_GNU_TYPE=arm-linux-gnueabihf\n\
not an environment variable assignment\n";
let mut env = HashMap::new();
parse_dpkg_architecture_output(output, &mut env);
assert_eq!(
env.get("DEB_HOST_GNU_TYPE").map(String::as_str),
Some("arm-linux-gnueabihf")
);
assert_eq!(
env.get("CROSS_COMPILE").map(String::as_str),
Some("arm-linux-gnueabihf-")
);
assert_eq!(env.len(), 2);
}
}
+122 -21
View File
@@ -3,6 +3,7 @@ use lazy_static::lazy_static;
use serde::Deserialize; use serde::Deserialize;
use std::error::Error; use std::error::Error;
use std::path::Path; use std::path::Path;
use std::time::Duration;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
/// Information about a specific distribution series /// Information about a specific distribution series
@@ -13,8 +14,8 @@ pub struct SeriesInformation {
pub codename: String, pub codename: String,
/// Series version as numbers /// Series version as numbers
pub version: Option<String>, pub version: Option<String>,
/// Series creation date /// Series creation date (absent if missing or invalid in the CSV data)
pub created: NaiveDate, pub created: Option<NaiveDate>,
/// Series release date /// Series release date
pub release: Option<NaiveDate>, pub release: Option<NaiveDate>,
/// Series end-of-life date /// Series end-of-life date
@@ -42,7 +43,45 @@ struct Data {
const DATA_YAML: &str = include_str!("../distro_info.yml"); const DATA_YAML: &str = include_str!("../distro_info.yml");
lazy_static! { lazy_static! {
static ref DATA: Data = serde_yaml::from_str(DATA_YAML).unwrap(); // The YAML is include_str!'d at compile time and statically valid; if it
// ever failed to parse it would be a build-time bug that cannot be
// recovered from at runtime, so panicking here is acceptable.
static ref DATA: Data = serde_yaml::from_str(DATA_YAML)
.expect("built-in distro_info.yml data is statically valid and must parse");
// Shared HTTP client used for all outgoing plain requests: timeouts keep
// a hanging remote (connect or transfer) from stalling pkh indefinitely.
static ref HTTP_CLIENT: reqwest::Client = reqwest::Client::builder()
.connect_timeout(Duration::from_secs(10))
.timeout(Duration::from_secs(30))
.build()
.expect("building the shared HTTP client with static options cannot fail");
}
/// Shared HTTP client with a connect timeout (10s) and a total request
/// timeout (30s), to be used for all outgoing plain HTTP(S) requests
pub(crate) fn http_client() -> &'static reqwest::Client {
&HTTP_CLIENT
}
/// Parse an optional '%Y-%m-%d' date from a CSV cell, warning instead of
/// panicking on invalid remote data
fn parse_optional_date(value: Option<&str>, series: &str, field: &str) -> Option<NaiveDate> {
value.and_then(
|date_str| match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
Ok(date) => Some(date),
Err(e) => {
log::warn!(
"Invalid '{}' date '{}' for series '{}': {}. Ignoring the date.",
field,
date_str,
series,
e
);
None
}
},
)
} }
fn parse_series_csv(content: &str) -> Result<Vec<SeriesInformation>, Box<dyn Error>> { fn parse_series_csv(content: &str) -> Result<Vec<SeriesInformation>, Box<dyn Error>> {
@@ -79,24 +118,39 @@ fn parse_series_csv(content: &str) -> Result<Vec<SeriesInformation>, Box<dyn Err
let mut series_info_list = Vec::new(); let mut series_info_list = Vec::new();
for result in rdr.records() { for result in rdr.records() {
let record = result?; let record = match result {
let series = record.get(series_idx).unwrap().to_string(); Ok(record) => record,
let codename = record.get(codename_idx).unwrap().to_string(); Err(e) => {
log::warn!("Skipping malformed series CSV row: {}", e);
continue;
}
};
// Rows missing essential identification fields are skipped: they
// cannot be used nor reported meaningfully. Dates, on the other
// hand, are all optional in the model, so a bad date keeps the row.
let Some(series) = record.get(series_idx).filter(|s| !s.is_empty()) else {
log::warn!(
"Skipping series CSV row without a 'series' value: {:?}",
record
);
continue;
};
let Some(codename) = record.get(codename_idx).filter(|s| !s.is_empty()) else {
log::warn!(
"Skipping series CSV row for series '{}' without a 'codename' value",
series
);
continue;
};
let version = record.get(version_idx).map(|s| s.to_string()); let version = record.get(version_idx).map(|s| s.to_string());
let created = record let created = parse_optional_date(record.get(created_idx), series, "created");
.get(created_idx) let release = parse_optional_date(record.get(release_idx), series, "release");
.map(|date_str| NaiveDate::parse_from_str(date_str, "%Y-%m-%d").unwrap()) let eol = parse_optional_date(record.get(eol_idx), series, "eol");
.unwrap();
let release = record
.get(release_idx)
.map(|date_str| NaiveDate::parse_from_str(date_str, "%Y-%m-%d").unwrap());
let eol = record
.get(eol_idx)
.map(|date_str| NaiveDate::parse_from_str(date_str, "%Y-%m-%d").unwrap());
series_info_list.push(SeriesInformation { series_info_list.push(SeriesInformation {
series, series: series.to_string(),
codename, codename: codename.to_string(),
version, version,
created, created,
release, release,
@@ -134,7 +188,9 @@ pub async fn get_ordered_series(dist: &str) -> Result<Vec<SeriesInformation>, Bo
) )
})? })?
} else { } else {
reqwest::get(series_info.network.as_str()) http_client()
.get(series_info.network.as_str())
.send()
.await? .await?
.text() .text()
.await? .await?
@@ -324,7 +380,7 @@ pub async fn get_components(
let url = get_release_url(base_url, series, pocket); let url = get_release_url(base_url, series, pocket);
log::debug!("Fetching Release file from: {}", url); log::debug!("Fetching Release file from: {}", url);
let content = reqwest::get(&url).await?.text().await?; let content = http_client().get(&url).send().await?.text().await?;
for line in content.lines() { for line in content.lines() {
if line.starts_with("Components:") if line.starts_with("Components:")
@@ -359,7 +415,9 @@ pub async fn get_debian_series_number(series: &str) -> Result<Option<String>, Bo
) )
})? })?
} else { } else {
reqwest::get(series_info.network.as_str()) http_client()
.get(series_info.network.as_str())
.send()
.await? .await?
.text() .text()
.await? .await?
@@ -395,6 +453,49 @@ pub async fn get_debian_series_number(series: &str) -> Result<Option<String>, Bo
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn test_parse_series_csv_malformed_rows() {
// A short row (missing 'codename') is skipped, a row with an invalid
// 'created' date is kept without a date, and invalid 'release'/'eol'
// dates become None: none of this may panic on remote data
let csv_data = "series,codename,version,created,release,eol\n\
noble,Noble N,24.04,2023-10-26,2024-04-25,2029-04-25\n\
lonely\n\
badbad,Bad B,1.0,not-a-date,2020-01-01,also-bad\n\
sid,sid,unstable,1999-01-01,,\n";
let series = parse_series_csv(csv_data).unwrap();
// Rows are returned most recent first (the parser reverses the list),
// with the malformed 'lonely' row skipped entirely
let names: Vec<&str> = series.iter().map(|s| s.series.as_str()).collect();
assert_eq!(names, vec!["sid", "badbad", "noble"]);
let noble = &series[2];
assert_eq!(noble.codename, "Noble N");
assert_eq!(noble.version.as_deref(), Some("24.04"));
assert_eq!(
noble.created,
Some(NaiveDate::from_ymd_opt(2023, 10, 26).unwrap())
);
assert_eq!(
noble.release,
Some(NaiveDate::from_ymd_opt(2024, 4, 25).unwrap())
);
assert_eq!(
noble.eol,
Some(NaiveDate::from_ymd_opt(2029, 4, 25).unwrap())
);
let badbad = &series[1];
assert_eq!(badbad.created, None);
assert_eq!(
badbad.release,
Some(NaiveDate::from_ymd_opt(2020, 1, 1).unwrap())
);
assert_eq!(badbad.eol, None);
}
#[test] #[test]
fn test_get_dist_pockets_order() { fn test_get_dist_pockets_order() {
// Without an explicit pocket, packages are searched in this order: // Without an explicit pocket, packages are searched in this order:
+104 -14
View File
@@ -190,6 +190,10 @@ impl Iterator for DebianSources {
type Item = PackageStanza; type Item = PackageStanza;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
// Iterate over stanzas in a loop: package-less (blank) stanzas are
// skipped without recursion, so that a crafted index with many
// consecutive blank stanzas cannot blow the stack
loop {
let stanza = self.splitted_sources.next()?; let stanza = self.splitted_sources.next()?;
// Parse stanza into a hashmap of strings, the fields // Parse stanza into a hashmap of strings, the fields
@@ -213,11 +217,22 @@ impl Iterator for DebianSources {
} }
} }
let pkg = fields.get("Package"); let Some(package) = fields.get("Package") else {
if pkg.is_none() {
// Skip empty stanza // Skip empty stanza
return self.next(); continue;
} };
let package = package.to_string();
// A stanza without a version is malformed remote data: skip it
// rather than panicking
let Some(version) = fields.get("Version") else {
debug!(
"Skipping malformed stanza for package '{}' without a 'Version' field",
package
);
continue;
};
let version = version.to_string();
// Parse package files. // Parse package files.
// Prefer the strongest available checksum field: Checksums-Sha256, // Prefer the strongest available checksum field: Checksums-Sha256,
@@ -254,9 +269,9 @@ impl Iterator for DebianSources {
vcs.split_whitespace().next().unwrap_or(vcs).to_string() vcs.split_whitespace().next().unwrap_or(vcs).to_string()
}); });
Some(PackageStanza { return Some(PackageStanza {
package: fields.get("Package").unwrap().to_string(), package,
version: fields.get("Version").unwrap().to_string(), version,
directory: fields.get("Directory").cloned().unwrap_or_default(), directory: fields.get("Directory").cloned().unwrap_or_default(),
format: fields format: fields
.get("Format") .get("Format")
@@ -265,7 +280,8 @@ impl Iterator for DebianSources {
vcs_git, vcs_git,
vcs_browser: fields.get("Vcs-Browser").cloned(), vcs_browser: fields.get("Vcs-Browser").cloned(),
files, files,
}) });
}
} }
} }
@@ -361,21 +377,28 @@ async fn get(
} }
debug!("Found components: {:?}", components); debug!("Found components: {:?}", components);
// Collect the failures of individual fetch attempts so that, if the
// package is not found, the final error explains what actually went
// wrong instead of misleadingly claiming a plain 'not found'
let mut fetch_errors: Vec<String> = Vec::new();
for component in components { for component in components {
let url = crate::distro_info::get_sources_url(&base_url, series, pocket, &component); let url = crate::distro_info::get_sources_url(&base_url, series, pocket, &component);
debug!("Fetching sources from: {}", url); debug!("Fetching sources from: {}", url);
let response = match reqwest::get(&url).await { let response = match crate::distro_info::http_client().get(&url).send().await {
Ok(resp) => resp, Ok(resp) => resp,
Err(e) => { Err(e) => {
debug!("Failed to fetch {}: {}", url, e); debug!("Failed to fetch {}: {}", url, e);
fetch_errors.push(format!("{suite}/{component}: {}", e));
continue; continue;
} }
}; };
if !response.status().is_success() { if !response.status().is_success() {
debug!("Failed to fetch {}: status {}", url, response.status()); debug!("Failed to fetch {}: status {}", url, response.status());
fetch_errors.push(format!("{suite}/{component}: HTTP {}", response.status()));
continue; continue;
} }
@@ -417,9 +440,14 @@ async fn get(
} }
} }
let details = if fetch_errors.is_empty() {
String::new()
} else {
format!(" (last errors: {})", fetch_errors.join("; "))
};
Err(format!( Err(format!(
"Package '{}' not found in {}/{}", "Package '{}' not found in {}/{}{}",
package_name, dist, series package_name, dist, series, details
) )
.into()) .into())
} }
@@ -436,6 +464,11 @@ async fn find_package(
) -> Result<PackageInfo, Box<dyn Error>> { ) -> Result<PackageInfo, Box<dyn Error>> {
let series_list = crate::distro_info::get_ordered_series_name(dist).await?; let series_list = crate::distro_info::get_ordered_series_name(dist).await?;
// Collect the failures of the individual series/pocket probes so that,
// if nothing is found, the final error summarizes what went wrong
// (e.g. network errors, HTTP statuses) instead of a bare 'not found'
let mut attempt_errors: Vec<String> = Vec::new();
for (i, series) in series_list.iter().enumerate() { for (i, series) in series_list.iter().enumerate() {
if let Some(cb) = progress { if let Some(cb) = progress {
cb("", &format!("Checking {}...", series), i, series_list.len()); cb("", &format!("Checking {}...", series), i, series_list.len());
@@ -486,13 +519,32 @@ async fn find_package(
if e.downcast_ref::<release::VerifyError>().is_some() { if e.downcast_ref::<release::VerifyError>().is_some() {
return Err(e); return Err(e);
} }
// Remember the failure for the final error message, and
// keep probing the other series/pockets
let suite = if p.is_empty() {
series.clone()
} else {
format!("{series}-{p}")
};
attempt_errors.push(format!("{}: {}", suite, e));
continue; continue;
} }
} }
} }
} }
Err(format!("Package '{}' not found.", package_name).into()) // Keep only the last few attempts so the message stays readable
if attempt_errors.len() > 5 {
let drain_to = attempt_errors.len() - 5;
attempt_errors.drain(..drain_to);
}
let details = if attempt_errors.is_empty() {
String::new()
} else {
format!(" (last errors: {})", attempt_errors.join("; "))
};
Err(format!("Package '{}' not found.{}", package_name, details).into())
} }
/// Fetch the 'Release' file at the root of a flat repository, and return its suite name /// Fetch the 'Release' file at the root of a flat repository, and return its suite name
@@ -502,7 +554,7 @@ async fn find_package(
/// read from the 'Codename' field, falling back to 'Suite'. /// read from the 'Codename' field, falling back to 'Suite'.
async fn get_flat_repo_series(base_url: &str) -> Result<String, Box<dyn Error>> { async fn get_flat_repo_series(base_url: &str) -> Result<String, Box<dyn Error>> {
let url = format!("{}/Release", base_url.trim_end_matches('/')); let url = format!("{}/Release", base_url.trim_end_matches('/'));
let response = reqwest::get(&url).await?; let response = crate::distro_info::http_client().get(&url).send().await?;
if !response.status().is_success() { if !response.status().is_success() {
return Err(format!( return Err(format!(
"No Release file at '{}' (HTTP {}) - is '{}' the suite URL of a flat repository? \ "No Release file at '{}' (HTTP {}) - is '{}' the suite URL of a flat repository? \
@@ -540,7 +592,7 @@ async fn get_flat_repo_sources(
let mut errors = Vec::new(); let mut errors = Vec::new();
for name in ["Sources.xz", "Sources.gz", "Sources"] { for name in ["Sources.xz", "Sources.gz", "Sources"] {
let url = format!("{base}/{name}"); let url = format!("{base}/{name}");
match reqwest::get(&url).await { match crate::distro_info::http_client().get(&url).send().await {
Ok(response) if response.status().is_success() => { Ok(response) if response.status().is_success() => {
let data = response.bytes().await?.to_vec(); let data = response.bytes().await?.to_vec();
@@ -860,6 +912,44 @@ Directory: pool/main/h/hello
assert_eq!(info.version, "1.0"); assert_eq!(info.version, "1.0");
} }
#[test]
fn test_parse_sources_many_blank_stanzas() {
// A crafted index with many consecutive package-less stanzas must be
// iterated without recursion: 100k blank stanzas would overflow the
// stack with the old recursive 'return self.next()' implementation
let mut data = String::new();
for _ in 0..100_000 {
data.push_str("Not-Really-Package: x\n\n");
}
data.push_str("Package: hello\nVersion: 1.0\n");
let info = parse_sources(data.as_bytes(), "hello", None)
.unwrap()
.unwrap();
assert_eq!(info.package, "hello");
assert_eq!(info.version, "1.0");
}
#[test]
fn test_parse_sources_stanza_without_version() {
// A stanza with a 'Package' but no 'Version' field is malformed
// remote data: it must be skipped rather than panic
let data = "Package: noversion\nDirectory: pool/main/n/noversion\n\n\
Package: hello\nVersion: 1.0\n";
let info = parse_sources(data.as_bytes(), "hello", None)
.unwrap()
.unwrap();
assert_eq!(info.package, "hello");
assert_eq!(info.version, "1.0");
assert!(
parse_sources(data.as_bytes(), "noversion", None)
.unwrap()
.is_none()
);
}
#[tokio::test] #[tokio::test]
async fn test_find_package_fallback() { async fn test_find_package_fallback() {
// python2.7 is in bullseye but not above // python2.7 is in bullseye but not above
+25 -7
View File
@@ -310,8 +310,14 @@ async fn download_file_checksum(
target_dir: &Path, target_dir: &Path,
progress: ProgressCallback<'_>, progress: ProgressCallback<'_>,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
// Download with reqwest // Download with the shared client (connect timeout). Large orig tarballs
let response = reqwest::get(url).await?; // can legitimately take longer than the client's default total timeout,
// so use a generous per-request timeout for streaming downloads
let response = crate::distro_info::http_client()
.get(url)
.timeout(std::time::Duration::from_secs(30 * 60))
.send()
.await?;
if !response.status().is_success() { if !response.status().is_success() {
return Err(format!("Failed to download '{}' : {}", url, response.status()).into()); return Err(format!("Failed to download '{}' : {}", url, response.status()).into());
} }
@@ -322,7 +328,12 @@ async fn download_file_checksum(
let mut index = 0; let mut index = 0;
// Target file: extract file name from URL // Target file: extract file name from URL
let filename = Path::new(url).file_name().unwrap().to_str().unwrap(); let filename = Path::new(url)
.file_name()
.and_then(|f| f.to_str())
.ok_or_else(|| {
format!("Could not determine a file name from URL '{url}' to download the package file")
})?;
let path = target_dir.join(filename); let path = target_dir.join(filename);
let mut file = File::create(path)?; let mut file = File::create(path)?;
@@ -650,10 +661,17 @@ pub async fn pull(
// we target the development branch, i.e. the default branch // we target the development branch, i.e. the default branch
// Only use Ubuntu-specific branch naming if the VCS is from Launchpad // Only use Ubuntu-specific branch naming if the VCS is from Launchpad
let is_launchpad_vcs = url.contains("launchpad.net"); let is_launchpad_vcs = url.contains("launchpad.net");
let branch_name = if crate::distro_info::get_ordered_series_name(package_info.dist.as_str()) let series_list =
.await?[0] crate::distro_info::get_ordered_series_name(package_info.dist.as_str()).await?;
!= *series let latest_series = series_list.first().ok_or_else(|| {
{ format!(
"No series information available for distribution '{}', \
cannot determine its development series to select the git branch. \
The 'distro-info' package provides this data.",
package_info.dist
)
})?;
let branch_name = if latest_series != series {
if package_info.dist == "ubuntu" && is_launchpad_vcs { if package_info.dist == "ubuntu" && is_launchpad_vcs {
Some(format!("{}/{}", package_info.dist, series)) Some(format!("{}/{}", package_info.dist, series))
} else { } else {