Handle malformed remote and edge-case data instead of panicking
- distro_info: malformed CSV rows are skipped with a warning, dates that fail to parse become None, and all plain HTTP requests go through a shared reqwest client with connect/total timeouts - package_info: the Sources stanza iterator is iterative (a crafted index with many blank stanzas overflowed the stack), stanzas missing a Version are skipped, and failed series/pocket probes are summarized in the final 'not found' error instead of being silently dropped - pull: no double unwrap on the remote-derived artifact filename, an empty series list is an error, and streaming downloads get a per-request timeout - deb/cross: dpkg-architecture output parsing skips unexpected lines and its exit status is checked, as is dpkg --add-architecture - changelog: version increments parse as u64 with checked arithmetic (1.0-20250123123456 used to panic on the u32 parse)
This commit is contained in:
+68
-23
@@ -40,7 +40,7 @@ pub fn generate_entry(
|
|||||||
version.to_string()
|
version.to_string()
|
||||||
} else {
|
} else {
|
||||||
// TODO: Pass these flags from CLI
|
// TODO: Pass these flags from CLI
|
||||||
compute_new_version(&old_version, false, false, false)
|
compute_new_version(&old_version, false, false, false)?
|
||||||
};
|
};
|
||||||
|
|
||||||
let (maintainer_name, maintainer_email) = get_maintainer_info()?;
|
let (maintainer_name, maintainer_email) = get_maintainer_info()?;
|
||||||
@@ -68,7 +68,7 @@ fn compute_new_version(
|
|||||||
is_ubuntu: bool,
|
is_ubuntu: bool,
|
||||||
is_rebuild: bool,
|
is_rebuild: bool,
|
||||||
is_nmu: bool,
|
is_nmu: bool,
|
||||||
) -> String {
|
) -> Result<String, Box<dyn std::error::Error>> {
|
||||||
if is_ubuntu {
|
if is_ubuntu {
|
||||||
return increment_suffix(old_version, "ubuntu");
|
return increment_suffix(old_version, "ubuntu");
|
||||||
}
|
}
|
||||||
@@ -86,7 +86,7 @@ fn compute_new_version(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Increment a version number by 1, for a given suffix
|
/// Increment a version number by 1, for a given suffix
|
||||||
fn increment_suffix(version: &str, suffix: &str) -> String {
|
fn increment_suffix(version: &str, suffix: &str) -> Result<String, Box<dyn std::error::Error>> {
|
||||||
// If suffix is empty, we just look for trailing digits
|
// If suffix is empty, we just look for trailing digits
|
||||||
// If suffix is not empty, we look for suffix followed by digits
|
// If suffix is not empty, we look for suffix followed by digits
|
||||||
|
|
||||||
@@ -100,19 +100,33 @@ fn increment_suffix(version: &str, suffix: &str) -> String {
|
|||||||
|
|
||||||
if let Some(caps) = re.captures(version) {
|
if let Some(caps) = re.captures(version) {
|
||||||
let num_str = caps.get(1).unwrap().as_str();
|
let num_str = caps.get(1).unwrap().as_str();
|
||||||
let num: u32 = num_str.parse().unwrap();
|
// Parse as u64 so that large trailing numbers (e.g. date-based
|
||||||
|
// versions like '1.0-20250123123456', which do not fit in a u32)
|
||||||
|
// still increment normally
|
||||||
|
let num: u64 = num_str.parse().map_err(|_| {
|
||||||
|
format!(
|
||||||
|
"Cannot increment version '{version}': trailing number '{num_str}' \
|
||||||
|
is too large to be incremented. Specify a version explicitly instead."
|
||||||
|
)
|
||||||
|
})?;
|
||||||
let range = caps.get(1).unwrap().range();
|
let range = caps.get(1).unwrap().range();
|
||||||
|
let new_num = num.checked_add(1).ok_or_else(|| {
|
||||||
|
format!(
|
||||||
|
"Cannot increment version '{version}': trailing number {num} \
|
||||||
|
is too large to be incremented. Specify a version explicitly instead."
|
||||||
|
)
|
||||||
|
})?;
|
||||||
let mut new_ver = version.to_string();
|
let mut new_ver = version.to_string();
|
||||||
new_ver.replace_range(range, &(num + 1).to_string());
|
new_ver.replace_range(range, &new_num.to_string());
|
||||||
return new_ver;
|
return Ok(new_ver);
|
||||||
}
|
}
|
||||||
|
|
||||||
// If pattern not found, append suffix + "1"
|
// If pattern not found, append suffix + "1"
|
||||||
// But if suffix is empty, we default to appending "-1" (standard Debian revision start)
|
// But if suffix is empty, we default to appending "-1" (standard Debian revision start)
|
||||||
if suffix.is_empty() {
|
if suffix.is_empty() {
|
||||||
format!("{}-1", version)
|
Ok(format!("{}-1", version))
|
||||||
} else {
|
} else {
|
||||||
format!("{}{}{}", version, suffix, 1)
|
Ok(format!("{}{}{}", version, suffix, 1))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -381,70 +395,101 @@ mod tests {
|
|||||||
fn test_compute_new_version() {
|
fn test_compute_new_version() {
|
||||||
// Debian upload
|
// Debian upload
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
compute_new_version("15.2.0-8", false, false, false),
|
compute_new_version("15.2.0-8", false, false, false).unwrap(),
|
||||||
"15.2.0-9"
|
"15.2.0-9"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
compute_new_version("15.2.0-9", false, false, false),
|
compute_new_version("15.2.0-9", false, false, false).unwrap(),
|
||||||
"15.2.0-10"
|
"15.2.0-10"
|
||||||
);
|
);
|
||||||
|
|
||||||
// Ubuntu upload
|
// Ubuntu upload
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
compute_new_version("15.2.0-9", true, false, false),
|
compute_new_version("15.2.0-9", true, false, false).unwrap(),
|
||||||
"15.2.0-9ubuntu1"
|
"15.2.0-9ubuntu1"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
compute_new_version("15.2.0-9ubuntu1", true, false, false),
|
compute_new_version("15.2.0-9ubuntu1", true, false, false).unwrap(),
|
||||||
"15.2.0-9ubuntu2"
|
"15.2.0-9ubuntu2"
|
||||||
);
|
);
|
||||||
|
|
||||||
// No change rebuild
|
// No change rebuild
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
compute_new_version("15.2.0-9", false, true, false),
|
compute_new_version("15.2.0-9", false, true, false).unwrap(),
|
||||||
"15.2.0-9build1"
|
"15.2.0-9build1"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
compute_new_version("15.2.0-9build1", false, true, false),
|
compute_new_version("15.2.0-9build1", false, true, false).unwrap(),
|
||||||
"15.2.0-9build2"
|
"15.2.0-9build2"
|
||||||
);
|
);
|
||||||
|
|
||||||
// Rebuild of Ubuntu version
|
// Rebuild of Ubuntu version
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
compute_new_version("15.2.0-9ubuntu1", false, true, false),
|
compute_new_version("15.2.0-9ubuntu1", false, true, false).unwrap(),
|
||||||
"15.2.0-9ubuntu1build1"
|
"15.2.0-9ubuntu1build1"
|
||||||
);
|
);
|
||||||
|
|
||||||
// NMU
|
// NMU
|
||||||
// Native
|
// Native
|
||||||
assert_eq!(compute_new_version("1.0", false, false, true), "1.0+nmu1");
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
compute_new_version("1.0+nmu1", false, false, true),
|
compute_new_version("1.0", false, false, true).unwrap(),
|
||||||
|
"1.0+nmu1"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
compute_new_version("1.0+nmu1", false, false, true).unwrap(),
|
||||||
"1.0+nmu2"
|
"1.0+nmu2"
|
||||||
);
|
);
|
||||||
|
|
||||||
// Non-native
|
// Non-native
|
||||||
assert_eq!(compute_new_version("1.0-1", false, false, true), "1.0-1.1");
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
compute_new_version("1.0-1.1", false, false, true),
|
compute_new_version("1.0-1", false, false, true).unwrap(),
|
||||||
|
"1.0-1.1"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
compute_new_version("1.0-1.1", false, false, true).unwrap(),
|
||||||
"1.0-1.2"
|
"1.0-1.2"
|
||||||
);
|
);
|
||||||
|
|
||||||
// NMU of NMU?
|
// NMU of NMU?
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
compute_new_version("1.0-1.2", false, false, true),
|
compute_new_version("1.0-1.2", false, false, true).unwrap(),
|
||||||
"1.0-1.3"
|
"1.0-1.3"
|
||||||
);
|
);
|
||||||
|
|
||||||
// Native package uploads
|
// Native package uploads
|
||||||
assert_eq!(compute_new_version("1.0", false, false, false), "1.1");
|
|
||||||
assert_eq!(compute_new_version("1.0.5", false, false, false), "1.0.6");
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
compute_new_version("20241126", false, false, false),
|
compute_new_version("1.0", false, false, false).unwrap(),
|
||||||
|
"1.1"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
compute_new_version("1.0.5", false, false, false).unwrap(),
|
||||||
|
"1.0.6"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
compute_new_version("20241126", false, false, false).unwrap(),
|
||||||
"20241127"
|
"20241127"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_compute_new_version_large_trailing_number() {
|
||||||
|
// Date-based versions with a trailing number larger than u32::MAX
|
||||||
|
// must increment normally (they fit in a u64)
|
||||||
|
assert_eq!(
|
||||||
|
compute_new_version("1.0-20250123123456", false, false, false).unwrap(),
|
||||||
|
"1.0-20250123123457"
|
||||||
|
);
|
||||||
|
|
||||||
|
// A number that does not even fit in a u64 yields a clear error
|
||||||
|
// instead of panicking
|
||||||
|
let err = compute_new_version("1.0-99999999999999999999999999", false, false, false);
|
||||||
|
assert!(err.is_err());
|
||||||
|
|
||||||
|
// u64::MAX itself cannot be incremented
|
||||||
|
let err = compute_new_version("1.0-18446744073709551615", false, false, false);
|
||||||
|
assert!(err.is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_get_maintainer_info() {
|
fn test_get_maintainer_info() {
|
||||||
// Test with env vars
|
// Test with env vars
|
||||||
|
|||||||
+110
-17
@@ -1,24 +1,19 @@
|
|||||||
use crate::context::Context;
|
use crate::context::Context;
|
||||||
|
use log::debug;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
/// Set environment variables for cross-compilation
|
/// Parse 'dpkg-architecture' output (KEY=value lines) into a set of
|
||||||
pub fn setup_environment(
|
/// environment variables. Unexpected lines (e.g. warnings on stderr leaking
|
||||||
env: &mut HashMap<String, String>,
|
/// into stdout) are skipped instead of causing a failure.
|
||||||
arch: &str,
|
fn parse_dpkg_architecture_output(output: &str, env: &mut HashMap<String, String>) {
|
||||||
ctx: Arc<Context>,
|
|
||||||
) -> Result<(), Box<dyn Error>> {
|
|
||||||
let dpkg_architecture = String::from_utf8(
|
|
||||||
ctx.command("dpkg-architecture")
|
|
||||||
.arg("-a")
|
|
||||||
.arg(arch)
|
|
||||||
.output()?
|
|
||||||
.stdout,
|
|
||||||
)?;
|
|
||||||
let env_var_regex = regex::Regex::new(r"(?<key>.*)=(?<value>.*)").unwrap();
|
let env_var_regex = regex::Regex::new(r"(?<key>.*)=(?<value>.*)").unwrap();
|
||||||
for l in dpkg_architecture.lines() {
|
for l in output.lines() {
|
||||||
let capture = env_var_regex.captures(l).unwrap();
|
let Some(capture) = env_var_regex.captures(l) else {
|
||||||
|
debug!("Skipping unexpected dpkg-architecture output line: '{l}'");
|
||||||
|
continue;
|
||||||
|
};
|
||||||
let key = capture.name("key").unwrap().as_str().to_string();
|
let key = capture.name("key").unwrap().as_str().to_string();
|
||||||
let value = capture.name("value").unwrap().as_str().to_string();
|
let value = capture.name("value").unwrap().as_str().to_string();
|
||||||
|
|
||||||
@@ -28,6 +23,45 @@ pub fn setup_environment(
|
|||||||
env.insert("CROSS_COMPILE".to_string(), format!("{value}-"));
|
env.insert("CROSS_COMPILE".to_string(), format!("{value}-"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set environment variables for cross-compilation
|
||||||
|
pub fn setup_environment(
|
||||||
|
env: &mut HashMap<String, String>,
|
||||||
|
arch: &str,
|
||||||
|
ctx: Arc<Context>,
|
||||||
|
) -> Result<(), Box<dyn Error>> {
|
||||||
|
let output = ctx
|
||||||
|
.command("dpkg-architecture")
|
||||||
|
.arg("-a")
|
||||||
|
.arg(arch)
|
||||||
|
.output()
|
||||||
|
.map_err(|e| {
|
||||||
|
format!(
|
||||||
|
"Failed to run 'dpkg-architecture -a {arch}': {e}. \
|
||||||
|
Is 'dpkg-dev' installed?"
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
if !output.status.success() {
|
||||||
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||||
|
return Err(format!(
|
||||||
|
"'dpkg-architecture -a {}' failed with status: {}.{}",
|
||||||
|
arch,
|
||||||
|
output.status,
|
||||||
|
if stderr.trim().is_empty() {
|
||||||
|
String::new()
|
||||||
|
} else {
|
||||||
|
format!("\ndpkg-architecture output:\n{}", stderr.trim())
|
||||||
|
}
|
||||||
|
)
|
||||||
|
.into());
|
||||||
|
}
|
||||||
|
|
||||||
|
let dpkg_architecture = String::from_utf8(output.stdout)
|
||||||
|
.map_err(|e| format!("Invalid UTF-8 in dpkg-architecture output: {e}"))?;
|
||||||
|
parse_dpkg_architecture_output(&dpkg_architecture, env);
|
||||||
|
|
||||||
env.insert("DEB_BUILD_PROFILES".to_string(), "cross".to_string());
|
env.insert("DEB_BUILD_PROFILES".to_string(), "cross".to_string());
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -44,10 +78,19 @@ pub fn ensure_repositories(
|
|||||||
let local_arch = crate::get_current_arch();
|
let local_arch = crate::get_current_arch();
|
||||||
|
|
||||||
// Add target ('host') architecture
|
// Add target ('host') architecture
|
||||||
ctx.command("dpkg")
|
let status = ctx
|
||||||
|
.command("dpkg")
|
||||||
.arg("--add-architecture")
|
.arg("--add-architecture")
|
||||||
.arg(arch)
|
.arg(arch)
|
||||||
.status()?;
|
.status()
|
||||||
|
.map_err(|e| format!("Failed to run 'dpkg --add-architecture {arch}': {e}"))?;
|
||||||
|
if !status.success() {
|
||||||
|
return Err(format!(
|
||||||
|
"'dpkg --add-architecture {}' failed with status: {}",
|
||||||
|
arch, status
|
||||||
|
)
|
||||||
|
.into());
|
||||||
|
}
|
||||||
|
|
||||||
// Check if we are on Ubuntu
|
// Check if we are on Ubuntu
|
||||||
let os_release = String::from_utf8(ctx.command("cat").arg("/etc/os-release").output()?.stdout)?;
|
let os_release = String::from_utf8(ctx.command("cat").arg("/etc/os-release").output()?.stdout)?;
|
||||||
@@ -142,3 +185,53 @@ pub fn ensure_repositories(
|
|||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_parse_dpkg_architecture_output() {
|
||||||
|
let output = "DEB_BUILD_ARCH=amd64\n\
|
||||||
|
DEB_HOST_ARCH=arm64\n\
|
||||||
|
DEB_HOST_GNU_TYPE=aarch64-linux-gnu\n";
|
||||||
|
|
||||||
|
let mut env = HashMap::new();
|
||||||
|
parse_dpkg_architecture_output(output, &mut env);
|
||||||
|
|
||||||
|
assert_eq!(env.get("DEB_BUILD_ARCH").map(String::as_str), Some("amd64"));
|
||||||
|
assert_eq!(env.get("DEB_HOST_ARCH").map(String::as_str), Some("arm64"));
|
||||||
|
assert_eq!(
|
||||||
|
env.get("DEB_HOST_GNU_TYPE").map(String::as_str),
|
||||||
|
Some("aarch64-linux-gnu")
|
||||||
|
);
|
||||||
|
// Derived variable for the GNU type
|
||||||
|
assert_eq!(
|
||||||
|
env.get("CROSS_COMPILE").map(String::as_str),
|
||||||
|
Some("aarch64-linux-gnu-")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_parse_dpkg_architecture_output_skips_unexpected_lines() {
|
||||||
|
// Unexpected lines (warnings on stdout, empty lines) must be skipped
|
||||||
|
// instead of panicking
|
||||||
|
let output = "dpkg-architecture: warning: something odd happened\n\
|
||||||
|
\n\
|
||||||
|
DEB_HOST_GNU_TYPE=arm-linux-gnueabihf\n\
|
||||||
|
not an environment variable assignment\n";
|
||||||
|
|
||||||
|
let mut env = HashMap::new();
|
||||||
|
parse_dpkg_architecture_output(output, &mut env);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
env.get("DEB_HOST_GNU_TYPE").map(String::as_str),
|
||||||
|
Some("arm-linux-gnueabihf")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
env.get("CROSS_COMPILE").map(String::as_str),
|
||||||
|
Some("arm-linux-gnueabihf-")
|
||||||
|
);
|
||||||
|
assert_eq!(env.len(), 2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+122
-21
@@ -3,6 +3,7 @@ use lazy_static::lazy_static;
|
|||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
/// Information about a specific distribution series
|
/// Information about a specific distribution series
|
||||||
@@ -13,8 +14,8 @@ pub struct SeriesInformation {
|
|||||||
pub codename: String,
|
pub codename: String,
|
||||||
/// Series version as numbers
|
/// Series version as numbers
|
||||||
pub version: Option<String>,
|
pub version: Option<String>,
|
||||||
/// Series creation date
|
/// Series creation date (absent if missing or invalid in the CSV data)
|
||||||
pub created: NaiveDate,
|
pub created: Option<NaiveDate>,
|
||||||
/// Series release date
|
/// Series release date
|
||||||
pub release: Option<NaiveDate>,
|
pub release: Option<NaiveDate>,
|
||||||
/// Series end-of-life date
|
/// Series end-of-life date
|
||||||
@@ -42,7 +43,45 @@ struct Data {
|
|||||||
|
|
||||||
const DATA_YAML: &str = include_str!("../distro_info.yml");
|
const DATA_YAML: &str = include_str!("../distro_info.yml");
|
||||||
lazy_static! {
|
lazy_static! {
|
||||||
static ref DATA: Data = serde_yaml::from_str(DATA_YAML).unwrap();
|
// The YAML is include_str!'d at compile time and statically valid; if it
|
||||||
|
// ever failed to parse it would be a build-time bug that cannot be
|
||||||
|
// recovered from at runtime, so panicking here is acceptable.
|
||||||
|
static ref DATA: Data = serde_yaml::from_str(DATA_YAML)
|
||||||
|
.expect("built-in distro_info.yml data is statically valid and must parse");
|
||||||
|
|
||||||
|
// Shared HTTP client used for all outgoing plain requests: timeouts keep
|
||||||
|
// a hanging remote (connect or transfer) from stalling pkh indefinitely.
|
||||||
|
static ref HTTP_CLIENT: reqwest::Client = reqwest::Client::builder()
|
||||||
|
.connect_timeout(Duration::from_secs(10))
|
||||||
|
.timeout(Duration::from_secs(30))
|
||||||
|
.build()
|
||||||
|
.expect("building the shared HTTP client with static options cannot fail");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shared HTTP client with a connect timeout (10s) and a total request
|
||||||
|
/// timeout (30s), to be used for all outgoing plain HTTP(S) requests
|
||||||
|
pub(crate) fn http_client() -> &'static reqwest::Client {
|
||||||
|
&HTTP_CLIENT
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse an optional '%Y-%m-%d' date from a CSV cell, warning instead of
|
||||||
|
/// panicking on invalid remote data
|
||||||
|
fn parse_optional_date(value: Option<&str>, series: &str, field: &str) -> Option<NaiveDate> {
|
||||||
|
value.and_then(
|
||||||
|
|date_str| match NaiveDate::parse_from_str(date_str, "%Y-%m-%d") {
|
||||||
|
Ok(date) => Some(date),
|
||||||
|
Err(e) => {
|
||||||
|
log::warn!(
|
||||||
|
"Invalid '{}' date '{}' for series '{}': {}. Ignoring the date.",
|
||||||
|
field,
|
||||||
|
date_str,
|
||||||
|
series,
|
||||||
|
e
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_series_csv(content: &str) -> Result<Vec<SeriesInformation>, Box<dyn Error>> {
|
fn parse_series_csv(content: &str) -> Result<Vec<SeriesInformation>, Box<dyn Error>> {
|
||||||
@@ -79,24 +118,39 @@ fn parse_series_csv(content: &str) -> Result<Vec<SeriesInformation>, Box<dyn Err
|
|||||||
let mut series_info_list = Vec::new();
|
let mut series_info_list = Vec::new();
|
||||||
|
|
||||||
for result in rdr.records() {
|
for result in rdr.records() {
|
||||||
let record = result?;
|
let record = match result {
|
||||||
let series = record.get(series_idx).unwrap().to_string();
|
Ok(record) => record,
|
||||||
let codename = record.get(codename_idx).unwrap().to_string();
|
Err(e) => {
|
||||||
|
log::warn!("Skipping malformed series CSV row: {}", e);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Rows missing essential identification fields are skipped: they
|
||||||
|
// cannot be used nor reported meaningfully. Dates, on the other
|
||||||
|
// hand, are all optional in the model, so a bad date keeps the row.
|
||||||
|
let Some(series) = record.get(series_idx).filter(|s| !s.is_empty()) else {
|
||||||
|
log::warn!(
|
||||||
|
"Skipping series CSV row without a 'series' value: {:?}",
|
||||||
|
record
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Some(codename) = record.get(codename_idx).filter(|s| !s.is_empty()) else {
|
||||||
|
log::warn!(
|
||||||
|
"Skipping series CSV row for series '{}' without a 'codename' value",
|
||||||
|
series
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
};
|
||||||
let version = record.get(version_idx).map(|s| s.to_string());
|
let version = record.get(version_idx).map(|s| s.to_string());
|
||||||
let created = record
|
let created = parse_optional_date(record.get(created_idx), series, "created");
|
||||||
.get(created_idx)
|
let release = parse_optional_date(record.get(release_idx), series, "release");
|
||||||
.map(|date_str| NaiveDate::parse_from_str(date_str, "%Y-%m-%d").unwrap())
|
let eol = parse_optional_date(record.get(eol_idx), series, "eol");
|
||||||
.unwrap();
|
|
||||||
let release = record
|
|
||||||
.get(release_idx)
|
|
||||||
.map(|date_str| NaiveDate::parse_from_str(date_str, "%Y-%m-%d").unwrap());
|
|
||||||
let eol = record
|
|
||||||
.get(eol_idx)
|
|
||||||
.map(|date_str| NaiveDate::parse_from_str(date_str, "%Y-%m-%d").unwrap());
|
|
||||||
|
|
||||||
series_info_list.push(SeriesInformation {
|
series_info_list.push(SeriesInformation {
|
||||||
series,
|
series: series.to_string(),
|
||||||
codename,
|
codename: codename.to_string(),
|
||||||
version,
|
version,
|
||||||
created,
|
created,
|
||||||
release,
|
release,
|
||||||
@@ -134,7 +188,9 @@ pub async fn get_ordered_series(dist: &str) -> Result<Vec<SeriesInformation>, Bo
|
|||||||
)
|
)
|
||||||
})?
|
})?
|
||||||
} else {
|
} else {
|
||||||
reqwest::get(series_info.network.as_str())
|
http_client()
|
||||||
|
.get(series_info.network.as_str())
|
||||||
|
.send()
|
||||||
.await?
|
.await?
|
||||||
.text()
|
.text()
|
||||||
.await?
|
.await?
|
||||||
@@ -324,7 +380,7 @@ pub async fn get_components(
|
|||||||
let url = get_release_url(base_url, series, pocket);
|
let url = get_release_url(base_url, series, pocket);
|
||||||
log::debug!("Fetching Release file from: {}", url);
|
log::debug!("Fetching Release file from: {}", url);
|
||||||
|
|
||||||
let content = reqwest::get(&url).await?.text().await?;
|
let content = http_client().get(&url).send().await?.text().await?;
|
||||||
|
|
||||||
for line in content.lines() {
|
for line in content.lines() {
|
||||||
if line.starts_with("Components:")
|
if line.starts_with("Components:")
|
||||||
@@ -359,7 +415,9 @@ pub async fn get_debian_series_number(series: &str) -> Result<Option<String>, Bo
|
|||||||
)
|
)
|
||||||
})?
|
})?
|
||||||
} else {
|
} else {
|
||||||
reqwest::get(series_info.network.as_str())
|
http_client()
|
||||||
|
.get(series_info.network.as_str())
|
||||||
|
.send()
|
||||||
.await?
|
.await?
|
||||||
.text()
|
.text()
|
||||||
.await?
|
.await?
|
||||||
@@ -395,6 +453,49 @@ pub async fn get_debian_series_number(series: &str) -> Result<Option<String>, Bo
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_parse_series_csv_malformed_rows() {
|
||||||
|
// A short row (missing 'codename') is skipped, a row with an invalid
|
||||||
|
// 'created' date is kept without a date, and invalid 'release'/'eol'
|
||||||
|
// dates become None: none of this may panic on remote data
|
||||||
|
let csv_data = "series,codename,version,created,release,eol\n\
|
||||||
|
noble,Noble N,24.04,2023-10-26,2024-04-25,2029-04-25\n\
|
||||||
|
lonely\n\
|
||||||
|
badbad,Bad B,1.0,not-a-date,2020-01-01,also-bad\n\
|
||||||
|
sid,sid,unstable,1999-01-01,,\n";
|
||||||
|
|
||||||
|
let series = parse_series_csv(csv_data).unwrap();
|
||||||
|
|
||||||
|
// Rows are returned most recent first (the parser reverses the list),
|
||||||
|
// with the malformed 'lonely' row skipped entirely
|
||||||
|
let names: Vec<&str> = series.iter().map(|s| s.series.as_str()).collect();
|
||||||
|
assert_eq!(names, vec!["sid", "badbad", "noble"]);
|
||||||
|
|
||||||
|
let noble = &series[2];
|
||||||
|
assert_eq!(noble.codename, "Noble N");
|
||||||
|
assert_eq!(noble.version.as_deref(), Some("24.04"));
|
||||||
|
assert_eq!(
|
||||||
|
noble.created,
|
||||||
|
Some(NaiveDate::from_ymd_opt(2023, 10, 26).unwrap())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
noble.release,
|
||||||
|
Some(NaiveDate::from_ymd_opt(2024, 4, 25).unwrap())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
noble.eol,
|
||||||
|
Some(NaiveDate::from_ymd_opt(2029, 4, 25).unwrap())
|
||||||
|
);
|
||||||
|
|
||||||
|
let badbad = &series[1];
|
||||||
|
assert_eq!(badbad.created, None);
|
||||||
|
assert_eq!(
|
||||||
|
badbad.release,
|
||||||
|
Some(NaiveDate::from_ymd_opt(2020, 1, 1).unwrap())
|
||||||
|
);
|
||||||
|
assert_eq!(badbad.eol, None);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_get_dist_pockets_order() {
|
fn test_get_dist_pockets_order() {
|
||||||
// Without an explicit pocket, packages are searched in this order:
|
// Without an explicit pocket, packages are searched in this order:
|
||||||
|
|||||||
+104
-14
@@ -190,6 +190,10 @@ impl Iterator for DebianSources {
|
|||||||
type Item = PackageStanza;
|
type Item = PackageStanza;
|
||||||
|
|
||||||
fn next(&mut self) -> Option<Self::Item> {
|
fn next(&mut self) -> Option<Self::Item> {
|
||||||
|
// Iterate over stanzas in a loop: package-less (blank) stanzas are
|
||||||
|
// skipped without recursion, so that a crafted index with many
|
||||||
|
// consecutive blank stanzas cannot blow the stack
|
||||||
|
loop {
|
||||||
let stanza = self.splitted_sources.next()?;
|
let stanza = self.splitted_sources.next()?;
|
||||||
|
|
||||||
// Parse stanza into a hashmap of strings, the fields
|
// Parse stanza into a hashmap of strings, the fields
|
||||||
@@ -213,11 +217,22 @@ impl Iterator for DebianSources {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let pkg = fields.get("Package");
|
let Some(package) = fields.get("Package") else {
|
||||||
if pkg.is_none() {
|
|
||||||
// Skip empty stanza
|
// Skip empty stanza
|
||||||
return self.next();
|
continue;
|
||||||
}
|
};
|
||||||
|
let package = package.to_string();
|
||||||
|
|
||||||
|
// A stanza without a version is malformed remote data: skip it
|
||||||
|
// rather than panicking
|
||||||
|
let Some(version) = fields.get("Version") else {
|
||||||
|
debug!(
|
||||||
|
"Skipping malformed stanza for package '{}' without a 'Version' field",
|
||||||
|
package
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let version = version.to_string();
|
||||||
|
|
||||||
// Parse package files.
|
// Parse package files.
|
||||||
// Prefer the strongest available checksum field: Checksums-Sha256,
|
// Prefer the strongest available checksum field: Checksums-Sha256,
|
||||||
@@ -254,9 +269,9 @@ impl Iterator for DebianSources {
|
|||||||
vcs.split_whitespace().next().unwrap_or(vcs).to_string()
|
vcs.split_whitespace().next().unwrap_or(vcs).to_string()
|
||||||
});
|
});
|
||||||
|
|
||||||
Some(PackageStanza {
|
return Some(PackageStanza {
|
||||||
package: fields.get("Package").unwrap().to_string(),
|
package,
|
||||||
version: fields.get("Version").unwrap().to_string(),
|
version,
|
||||||
directory: fields.get("Directory").cloned().unwrap_or_default(),
|
directory: fields.get("Directory").cloned().unwrap_or_default(),
|
||||||
format: fields
|
format: fields
|
||||||
.get("Format")
|
.get("Format")
|
||||||
@@ -265,7 +280,8 @@ impl Iterator for DebianSources {
|
|||||||
vcs_git,
|
vcs_git,
|
||||||
vcs_browser: fields.get("Vcs-Browser").cloned(),
|
vcs_browser: fields.get("Vcs-Browser").cloned(),
|
||||||
files,
|
files,
|
||||||
})
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -361,21 +377,28 @@ async fn get(
|
|||||||
}
|
}
|
||||||
debug!("Found components: {:?}", components);
|
debug!("Found components: {:?}", components);
|
||||||
|
|
||||||
|
// Collect the failures of individual fetch attempts so that, if the
|
||||||
|
// package is not found, the final error explains what actually went
|
||||||
|
// wrong instead of misleadingly claiming a plain 'not found'
|
||||||
|
let mut fetch_errors: Vec<String> = Vec::new();
|
||||||
|
|
||||||
for component in components {
|
for component in components {
|
||||||
let url = crate::distro_info::get_sources_url(&base_url, series, pocket, &component);
|
let url = crate::distro_info::get_sources_url(&base_url, series, pocket, &component);
|
||||||
|
|
||||||
debug!("Fetching sources from: {}", url);
|
debug!("Fetching sources from: {}", url);
|
||||||
|
|
||||||
let response = match reqwest::get(&url).await {
|
let response = match crate::distro_info::http_client().get(&url).send().await {
|
||||||
Ok(resp) => resp,
|
Ok(resp) => resp,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
debug!("Failed to fetch {}: {}", url, e);
|
debug!("Failed to fetch {}: {}", url, e);
|
||||||
|
fetch_errors.push(format!("{suite}/{component}: {}", e));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if !response.status().is_success() {
|
if !response.status().is_success() {
|
||||||
debug!("Failed to fetch {}: status {}", url, response.status());
|
debug!("Failed to fetch {}: status {}", url, response.status());
|
||||||
|
fetch_errors.push(format!("{suite}/{component}: HTTP {}", response.status()));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -417,9 +440,14 @@ async fn get(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let details = if fetch_errors.is_empty() {
|
||||||
|
String::new()
|
||||||
|
} else {
|
||||||
|
format!(" (last errors: {})", fetch_errors.join("; "))
|
||||||
|
};
|
||||||
Err(format!(
|
Err(format!(
|
||||||
"Package '{}' not found in {}/{}",
|
"Package '{}' not found in {}/{}{}",
|
||||||
package_name, dist, series
|
package_name, dist, series, details
|
||||||
)
|
)
|
||||||
.into())
|
.into())
|
||||||
}
|
}
|
||||||
@@ -436,6 +464,11 @@ async fn find_package(
|
|||||||
) -> Result<PackageInfo, Box<dyn Error>> {
|
) -> Result<PackageInfo, Box<dyn Error>> {
|
||||||
let series_list = crate::distro_info::get_ordered_series_name(dist).await?;
|
let series_list = crate::distro_info::get_ordered_series_name(dist).await?;
|
||||||
|
|
||||||
|
// Collect the failures of the individual series/pocket probes so that,
|
||||||
|
// if nothing is found, the final error summarizes what went wrong
|
||||||
|
// (e.g. network errors, HTTP statuses) instead of a bare 'not found'
|
||||||
|
let mut attempt_errors: Vec<String> = Vec::new();
|
||||||
|
|
||||||
for (i, series) in series_list.iter().enumerate() {
|
for (i, series) in series_list.iter().enumerate() {
|
||||||
if let Some(cb) = progress {
|
if let Some(cb) = progress {
|
||||||
cb("", &format!("Checking {}...", series), i, series_list.len());
|
cb("", &format!("Checking {}...", series), i, series_list.len());
|
||||||
@@ -486,13 +519,32 @@ async fn find_package(
|
|||||||
if e.downcast_ref::<release::VerifyError>().is_some() {
|
if e.downcast_ref::<release::VerifyError>().is_some() {
|
||||||
return Err(e);
|
return Err(e);
|
||||||
}
|
}
|
||||||
|
// Remember the failure for the final error message, and
|
||||||
|
// keep probing the other series/pockets
|
||||||
|
let suite = if p.is_empty() {
|
||||||
|
series.clone()
|
||||||
|
} else {
|
||||||
|
format!("{series}-{p}")
|
||||||
|
};
|
||||||
|
attempt_errors.push(format!("{}: {}", suite, e));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Err(format!("Package '{}' not found.", package_name).into())
|
// Keep only the last few attempts so the message stays readable
|
||||||
|
if attempt_errors.len() > 5 {
|
||||||
|
let drain_to = attempt_errors.len() - 5;
|
||||||
|
attempt_errors.drain(..drain_to);
|
||||||
|
}
|
||||||
|
let details = if attempt_errors.is_empty() {
|
||||||
|
String::new()
|
||||||
|
} else {
|
||||||
|
format!(" (last errors: {})", attempt_errors.join("; "))
|
||||||
|
};
|
||||||
|
|
||||||
|
Err(format!("Package '{}' not found.{}", package_name, details).into())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fetch the 'Release' file at the root of a flat repository, and return its suite name
|
/// Fetch the 'Release' file at the root of a flat repository, and return its suite name
|
||||||
@@ -502,7 +554,7 @@ async fn find_package(
|
|||||||
/// read from the 'Codename' field, falling back to 'Suite'.
|
/// read from the 'Codename' field, falling back to 'Suite'.
|
||||||
async fn get_flat_repo_series(base_url: &str) -> Result<String, Box<dyn Error>> {
|
async fn get_flat_repo_series(base_url: &str) -> Result<String, Box<dyn Error>> {
|
||||||
let url = format!("{}/Release", base_url.trim_end_matches('/'));
|
let url = format!("{}/Release", base_url.trim_end_matches('/'));
|
||||||
let response = reqwest::get(&url).await?;
|
let response = crate::distro_info::http_client().get(&url).send().await?;
|
||||||
if !response.status().is_success() {
|
if !response.status().is_success() {
|
||||||
return Err(format!(
|
return Err(format!(
|
||||||
"No Release file at '{}' (HTTP {}) - is '{}' the suite URL of a flat repository? \
|
"No Release file at '{}' (HTTP {}) - is '{}' the suite URL of a flat repository? \
|
||||||
@@ -540,7 +592,7 @@ async fn get_flat_repo_sources(
|
|||||||
let mut errors = Vec::new();
|
let mut errors = Vec::new();
|
||||||
for name in ["Sources.xz", "Sources.gz", "Sources"] {
|
for name in ["Sources.xz", "Sources.gz", "Sources"] {
|
||||||
let url = format!("{base}/{name}");
|
let url = format!("{base}/{name}");
|
||||||
match reqwest::get(&url).await {
|
match crate::distro_info::http_client().get(&url).send().await {
|
||||||
Ok(response) if response.status().is_success() => {
|
Ok(response) if response.status().is_success() => {
|
||||||
let data = response.bytes().await?.to_vec();
|
let data = response.bytes().await?.to_vec();
|
||||||
|
|
||||||
@@ -860,6 +912,44 @@ Directory: pool/main/h/hello
|
|||||||
assert_eq!(info.version, "1.0");
|
assert_eq!(info.version, "1.0");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_parse_sources_many_blank_stanzas() {
|
||||||
|
// A crafted index with many consecutive package-less stanzas must be
|
||||||
|
// iterated without recursion: 100k blank stanzas would overflow the
|
||||||
|
// stack with the old recursive 'return self.next()' implementation
|
||||||
|
let mut data = String::new();
|
||||||
|
for _ in 0..100_000 {
|
||||||
|
data.push_str("Not-Really-Package: x\n\n");
|
||||||
|
}
|
||||||
|
data.push_str("Package: hello\nVersion: 1.0\n");
|
||||||
|
|
||||||
|
let info = parse_sources(data.as_bytes(), "hello", None)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(info.package, "hello");
|
||||||
|
assert_eq!(info.version, "1.0");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_parse_sources_stanza_without_version() {
|
||||||
|
// A stanza with a 'Package' but no 'Version' field is malformed
|
||||||
|
// remote data: it must be skipped rather than panic
|
||||||
|
let data = "Package: noversion\nDirectory: pool/main/n/noversion\n\n\
|
||||||
|
Package: hello\nVersion: 1.0\n";
|
||||||
|
|
||||||
|
let info = parse_sources(data.as_bytes(), "hello", None)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(info.package, "hello");
|
||||||
|
assert_eq!(info.version, "1.0");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
parse_sources(data.as_bytes(), "noversion", None)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_find_package_fallback() {
|
async fn test_find_package_fallback() {
|
||||||
// python2.7 is in bullseye but not above
|
// python2.7 is in bullseye but not above
|
||||||
|
|||||||
+25
-7
@@ -310,8 +310,14 @@ async fn download_file_checksum(
|
|||||||
target_dir: &Path,
|
target_dir: &Path,
|
||||||
progress: ProgressCallback<'_>,
|
progress: ProgressCallback<'_>,
|
||||||
) -> Result<(), Box<dyn Error>> {
|
) -> Result<(), Box<dyn Error>> {
|
||||||
// Download with reqwest
|
// Download with the shared client (connect timeout). Large orig tarballs
|
||||||
let response = reqwest::get(url).await?;
|
// can legitimately take longer than the client's default total timeout,
|
||||||
|
// so use a generous per-request timeout for streaming downloads
|
||||||
|
let response = crate::distro_info::http_client()
|
||||||
|
.get(url)
|
||||||
|
.timeout(std::time::Duration::from_secs(30 * 60))
|
||||||
|
.send()
|
||||||
|
.await?;
|
||||||
if !response.status().is_success() {
|
if !response.status().is_success() {
|
||||||
return Err(format!("Failed to download '{}' : {}", url, response.status()).into());
|
return Err(format!("Failed to download '{}' : {}", url, response.status()).into());
|
||||||
}
|
}
|
||||||
@@ -322,7 +328,12 @@ async fn download_file_checksum(
|
|||||||
let mut index = 0;
|
let mut index = 0;
|
||||||
|
|
||||||
// Target file: extract file name from URL
|
// Target file: extract file name from URL
|
||||||
let filename = Path::new(url).file_name().unwrap().to_str().unwrap();
|
let filename = Path::new(url)
|
||||||
|
.file_name()
|
||||||
|
.and_then(|f| f.to_str())
|
||||||
|
.ok_or_else(|| {
|
||||||
|
format!("Could not determine a file name from URL '{url}' to download the package file")
|
||||||
|
})?;
|
||||||
let path = target_dir.join(filename);
|
let path = target_dir.join(filename);
|
||||||
let mut file = File::create(path)?;
|
let mut file = File::create(path)?;
|
||||||
|
|
||||||
@@ -650,10 +661,17 @@ pub async fn pull(
|
|||||||
// we target the development branch, i.e. the default branch
|
// we target the development branch, i.e. the default branch
|
||||||
// Only use Ubuntu-specific branch naming if the VCS is from Launchpad
|
// Only use Ubuntu-specific branch naming if the VCS is from Launchpad
|
||||||
let is_launchpad_vcs = url.contains("launchpad.net");
|
let is_launchpad_vcs = url.contains("launchpad.net");
|
||||||
let branch_name = if crate::distro_info::get_ordered_series_name(package_info.dist.as_str())
|
let series_list =
|
||||||
.await?[0]
|
crate::distro_info::get_ordered_series_name(package_info.dist.as_str()).await?;
|
||||||
!= *series
|
let latest_series = series_list.first().ok_or_else(|| {
|
||||||
{
|
format!(
|
||||||
|
"No series information available for distribution '{}', \
|
||||||
|
cannot determine its development series to select the git branch. \
|
||||||
|
The 'distro-info' package provides this data.",
|
||||||
|
package_info.dist
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let branch_name = if latest_series != series {
|
||||||
if package_info.dist == "ubuntu" && is_launchpad_vcs {
|
if package_info.dist == "ubuntu" && is_launchpad_vcs {
|
||||||
Some(format!("{}/{}", package_info.dist, series))
|
Some(format!("{}/{}", package_info.dist, series))
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
Reference in New Issue
Block a user