vhaudiquet
4fae02bc85
The location embedded in the log record was styled with crossterm: a remote consumer of pkh's log records would receive ANSI codes inside the message text. Plain text is the logger's business to style.
1163 lines
42 KiB
Rust
1163 lines
42 KiB
Rust
use flate2::read::GzDecoder;
|
|
use std::collections::HashMap;
|
|
use std::error::Error;
|
|
use std::io::Read;
|
|
use xz2::read::XzDecoder;
|
|
|
|
use crate::ProgressCallback;
|
|
use crate::apt::release::{self, VerifiedRelease};
|
|
use log::{debug, warn};
|
|
|
|
/// Split a PPA reference into its `(user, name)` parts
|
|
///
|
|
/// A PPA is written `user/ppa_name` (e.g. `user/my-ppa`); anything else —
|
|
/// more segments, empty parts — is a format error carrying the canonical
|
|
/// message.
|
|
pub fn split_ppa(ppa: &str) -> Result<(&str, &str), String> {
|
|
let parts: Vec<&str> = ppa.split('/').collect();
|
|
if parts.len() == 2 && !parts[0].is_empty() && !parts[1].is_empty() {
|
|
Ok((parts[0], parts[1]))
|
|
} else {
|
|
Err(format!(
|
|
"Invalid PPA format: '{ppa}'. Expected: user/ppa_name"
|
|
))
|
|
}
|
|
}
|
|
|
|
/// Convert a PPA specification to a base URL
|
|
///
|
|
/// # Arguments
|
|
/// * user: user for the PPA
|
|
/// * name: name of the PPA
|
|
///
|
|
/// # Returns
|
|
/// * The base URL for the PPA (e.g., "https://ppa.launchpadcontent.net/user/ppa_name/ubuntu/")
|
|
pub fn ppa_to_base_url(user: &str, name: &str) -> String {
|
|
crate::launchpad::ppa_content_url(user, name)
|
|
}
|
|
|
|
fn check_launchpad_repo_sync(package: &str) -> Result<Option<String>, String> {
|
|
let url = crate::launchpad::ubuntu_source_git_url(package);
|
|
|
|
// Use libgit2 to check if the remote repository exists
|
|
// This is more reliable than HTTP HEAD requests when CGIt is disabled
|
|
match git2::Remote::create_detached(url.clone()) {
|
|
Ok(mut remote) => match remote.connect(git2::Direction::Fetch) {
|
|
Ok(_) => Ok(Some(url)),
|
|
Err(_) => Ok(None),
|
|
},
|
|
Err(_) => Ok(None),
|
|
}
|
|
}
|
|
|
|
/// Check if a Launchpad git repository exists for the given package.
|
|
///
|
|
/// This runs the blocking git2 connection check on a dedicated thread with a
|
|
/// timeout so that it never stalls the tokio runtime (the git2 connect call
|
|
/// has no built-in timeout and can hang for minutes on unreachable hosts).
|
|
async fn check_launchpad_repo(
|
|
package: &str,
|
|
) -> Result<Option<String>, Box<dyn Error + Send + Sync>> {
|
|
let package_owned = package.to_string();
|
|
let package_for_err = package_owned.clone();
|
|
let result = tokio::time::timeout(
|
|
std::time::Duration::from_secs(15),
|
|
tokio::task::spawn_blocking(move || check_launchpad_repo_sync(&package_owned)),
|
|
)
|
|
.await
|
|
.map_err(|_| {
|
|
format!(
|
|
"Timed out checking Launchpad repository for '{}'",
|
|
package_for_err
|
|
)
|
|
})?
|
|
.map_err(|e| format!("Launchpad check task failed: {}", e))?;
|
|
result.map_err(|e| -> Box<dyn Error + Send + Sync> { e.into() })
|
|
}
|
|
|
|
/// A File used in a source package
|
|
#[derive(Debug, Clone)]
|
|
pub struct FileEntry {
|
|
/// Name of the file
|
|
pub name: String,
|
|
/// Size of the file
|
|
pub size: u64,
|
|
/// Checksum hash for the file
|
|
pub checksum: String,
|
|
/// Algorithm used for the checksum
|
|
pub checksum_algo: ChecksumAlgo,
|
|
}
|
|
|
|
/// Checksum algorithm used for a file entry
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum ChecksumAlgo {
|
|
/// MD5 (legacy 'Files' field)
|
|
Md5,
|
|
/// SHA-256 ('Checksums-Sha256' field)
|
|
Sha256,
|
|
/// SHA-512 ('Checksums-Sha512' field)
|
|
Sha512,
|
|
}
|
|
|
|
impl ChecksumAlgo {
|
|
/// Compute the hex-encoded digest of the given data using this algorithm
|
|
pub fn hex_digest(&self, data: &[u8]) -> String {
|
|
match self {
|
|
ChecksumAlgo::Md5 => {
|
|
use md5::{Digest, Md5};
|
|
let mut hasher = Md5::new();
|
|
hasher.update(data);
|
|
hex::encode(hasher.finalize())
|
|
}
|
|
ChecksumAlgo::Sha256 => {
|
|
use sha2::{Digest, Sha256};
|
|
let mut hasher = Sha256::new();
|
|
hasher.update(data);
|
|
hex::encode(hasher.finalize())
|
|
}
|
|
ChecksumAlgo::Sha512 => {
|
|
use sha2::{Digest, Sha512};
|
|
let mut hasher = Sha512::new();
|
|
hasher.update(data);
|
|
hex::encode(hasher.finalize())
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A package 'stanza' as found is 'Sources.gz' files, containing basic information about a source package
|
|
#[derive(Debug)]
|
|
pub struct PackageStanza {
|
|
/// Name of the package
|
|
pub package: String,
|
|
/// Version number for the package
|
|
pub version: String,
|
|
/// Directory field in the stanza
|
|
pub directory: String,
|
|
/// Source package format (e.g. '3.0 (quilt)')
|
|
pub format: String,
|
|
/// Vcs-Git field in the stanza
|
|
pub vcs_git: Option<String>,
|
|
/// Vcs-Browser field in the stanza
|
|
pub vcs_browser: Option<String>,
|
|
/// Files present in the source package
|
|
pub files: Vec<FileEntry>,
|
|
}
|
|
|
|
/// Source package information
|
|
#[derive(Debug)]
|
|
pub struct PackageInfo {
|
|
/// Source 'stanza' for the package, containing basic information
|
|
pub stanza: PackageStanza,
|
|
/// Distribution for the package
|
|
pub dist: String,
|
|
/// Distribution series for the package
|
|
pub series: String,
|
|
/// Preferred VCS for the source package
|
|
///
|
|
/// Should be Launchpad on Ubuntu, and Salsa on Debian
|
|
pub preferred_vcs: Option<String>,
|
|
/// URL for the files of the source package
|
|
pub archive_url: String,
|
|
}
|
|
|
|
impl PackageInfo {
|
|
/// Returns true if the package is a Debian native package (no orig)
|
|
pub fn is_native(&self) -> bool {
|
|
self.stanza.format.contains("(native)")
|
|
}
|
|
}
|
|
|
|
/// Decompress a package index payload, detecting the compression format from
|
|
/// its magic bytes: gzip ('1f 8b'), xz ('fd 7zXZ 00'), or plain text
|
|
fn decompress_index(data: &[u8]) -> Result<String, Box<dyn Error>> {
|
|
let mut decoded: Box<dyn Read> = if data.starts_with(&[0x1f, 0x8b]) {
|
|
Box::new(GzDecoder::new(data))
|
|
} else if data.starts_with(&[0xfd, 0x37, b'z', b'X', b'Z', 0x00]) {
|
|
Box::new(XzDecoder::new(data))
|
|
} else {
|
|
return Ok(String::from_utf8(data.to_vec())?);
|
|
};
|
|
|
|
let mut s = String::new();
|
|
decoded.read_to_string(&mut s)?;
|
|
Ok(s)
|
|
}
|
|
|
|
struct DebianSources {
|
|
splitted_sources: std::str::Split<'static, &'static str>,
|
|
}
|
|
impl DebianSources {
|
|
fn new(data: &[u8]) -> Result<DebianSources, Box<dyn Error>> {
|
|
// Decode the index into a string, and split it on stanzas
|
|
let s = decompress_index(data)?;
|
|
|
|
// Convert the string to a static lifetime by leaking it
|
|
let static_str = Box::leak(s.into_boxed_str());
|
|
let splitted = static_str.split("\n\n");
|
|
|
|
Ok(DebianSources {
|
|
splitted_sources: splitted,
|
|
})
|
|
}
|
|
}
|
|
impl Iterator for DebianSources {
|
|
type Item = PackageStanza;
|
|
|
|
fn next(&mut self) -> Option<Self::Item> {
|
|
// Iterate over stanzas in a loop: package-less (blank) stanzas are
|
|
// skipped without recursion, so that a crafted index with many
|
|
// consecutive blank stanzas cannot blow the stack
|
|
loop {
|
|
let stanza = self.splitted_sources.next()?;
|
|
|
|
// Parse stanza into a hashmap of strings, the fields
|
|
let mut fields: HashMap<String, String> = HashMap::new();
|
|
let mut current_key = String::new();
|
|
|
|
for line in stanza.lines() {
|
|
if line.is_empty() {
|
|
continue;
|
|
}
|
|
|
|
if line.starts_with(' ') || line.starts_with('\t') {
|
|
// Continuation line
|
|
if let Some(val) = fields.get_mut(¤t_key) {
|
|
val.push('\n');
|
|
val.push_str(line.trim());
|
|
}
|
|
} else if let Some((key, value)) = line.split_once(':') {
|
|
current_key = key.trim().to_string();
|
|
fields.insert(current_key.clone(), value.trim().to_string());
|
|
}
|
|
}
|
|
|
|
let Some(package) = fields.get("Package") else {
|
|
// Skip empty stanza
|
|
continue;
|
|
};
|
|
let package = package.to_string();
|
|
|
|
// A stanza without a version is malformed remote data: skip it
|
|
// rather than panicking
|
|
let Some(version) = fields.get("Version") else {
|
|
debug!(
|
|
"Skipping malformed stanza for package '{}' without a 'Version' field",
|
|
package
|
|
);
|
|
continue;
|
|
};
|
|
let version = version.to_string();
|
|
|
|
// Parse package files.
|
|
// Prefer the strongest available checksum field: Checksums-Sha256,
|
|
// then Checksums-Sha512, then the legacy 'Files' (MD5) field.
|
|
// Some archives (e.g. the Ubuntu development series) no longer ship
|
|
// Checksums-Sha256, so falling back is required to keep working.
|
|
let mut files = Vec::new();
|
|
let (checksum_field, algo) = if fields.contains_key("Checksums-Sha256") {
|
|
("Checksums-Sha256", ChecksumAlgo::Sha256)
|
|
} else if fields.contains_key("Checksums-Sha512") {
|
|
("Checksums-Sha512", ChecksumAlgo::Sha512)
|
|
} else {
|
|
("Files", ChecksumAlgo::Md5)
|
|
};
|
|
if let Some(checksums) = fields.get(checksum_field) {
|
|
for line in checksums.lines() {
|
|
let parts: Vec<&str> = line.split_whitespace().collect();
|
|
if parts.len() >= 3 {
|
|
files.push(FileEntry {
|
|
checksum: parts[0].to_string(),
|
|
size: parts[1].parse().unwrap_or(0),
|
|
name: parts[2].to_string(),
|
|
checksum_algo: algo,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
// Parse Vcs-Git field: it may contain just a URL, or URL followed by -b <branch>
|
|
// e.g., "https://salsa.debian.org/science-team/paraview.git -b debian/latest"
|
|
let vcs_git = fields.get("Vcs-Git").map(|vcs| {
|
|
// Split on whitespace and take the first part (the URL)
|
|
// The URL should not contain spaces, so this is safe
|
|
vcs.split_whitespace().next().unwrap_or(vcs).to_string()
|
|
});
|
|
|
|
return Some(PackageStanza {
|
|
package,
|
|
version,
|
|
directory: fields.get("Directory").cloned().unwrap_or_default(),
|
|
format: fields
|
|
.get("Format")
|
|
.cloned()
|
|
.unwrap_or_else(|| "1.0".to_string()),
|
|
vcs_git,
|
|
vcs_browser: fields.get("Vcs-Browser").cloned(),
|
|
files,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Parse a 'Sources.gz' debian package file data, to look for a target package and
|
|
/// return the data for that package stanza
|
|
fn parse_sources(
|
|
data: &[u8],
|
|
target_package: &str,
|
|
target_version: Option<&str>,
|
|
) -> Result<Option<PackageStanza>, Box<dyn Error>> {
|
|
let mut sources = DebianSources::new(data)?;
|
|
|
|
// Find the right package, with the right version if requested
|
|
Ok(sources.find(|s| {
|
|
s.package == target_package
|
|
&& (target_version.is_none() || s.version == target_version.unwrap())
|
|
}))
|
|
}
|
|
|
|
/// Get package information from a package, distribution series, and pocket
|
|
///
|
|
/// `launchpad_vcs` is a pre-computed Launchpad VCS URL (if any), checked once
|
|
/// by the caller so that the blocking git2 connection test is not repeated for
|
|
/// every series/pocket combination.
|
|
async fn get(
|
|
package_name: &str,
|
|
series: &str,
|
|
pocket: &str,
|
|
version: Option<&str>,
|
|
base_url: Option<&str>,
|
|
launchpad_vcs: Option<String>,
|
|
) -> Result<PackageInfo, Box<dyn Error>> {
|
|
let dist = crate::distro_info::get_dist_from_series(series).await?;
|
|
|
|
// Use the pre-computed Launchpad VCS URL (if any).
|
|
// This is checked once in the caller rather than per-series to avoid
|
|
// repeated blocking git2 connections.
|
|
let mut preferred_vcs = launchpad_vcs;
|
|
|
|
// Determine the base URL to use (either provided PPA URL or default archive)
|
|
let distro_base_url = crate::distro_info::get_base_url(&dist)?;
|
|
let base_url = if let Some(ppa_url) = base_url {
|
|
ppa_url.to_string()
|
|
} else {
|
|
distro_base_url.clone()
|
|
};
|
|
|
|
// If using a custom base URL (PPA), disable VCS lookup to force archive download
|
|
let from_ppa = base_url != distro_base_url;
|
|
|
|
// Authenticate the metadata of the suite before trusting any index
|
|
// fetched from it: the signed Release file (InRelease, or Release plus
|
|
// detached Release.gpg) is verified against the archive keyring (or the
|
|
// PPA signing key), and each Sources index downloaded below is
|
|
// checksum-checked against it before parsing.
|
|
let suite = if pocket.is_empty() {
|
|
series.to_string()
|
|
} else {
|
|
format!("{series}-{pocket}")
|
|
};
|
|
let suite_url = format!("{base_url}/dists/{suite}");
|
|
|
|
let keyring_source = if from_ppa {
|
|
release::KeyringSource::Ppa {
|
|
base_url: base_url.clone(),
|
|
}
|
|
} else {
|
|
release::KeyringSource::Distro {
|
|
series: series.to_string(),
|
|
}
|
|
};
|
|
|
|
debug!("Verifying the Release file of: {}", suite_url);
|
|
let verified = match release::verify_suite(&suite_url, keyring_source, true).await {
|
|
Ok(release::Verification::Available(verified)) => verified,
|
|
Ok(release::Verification::Unavailable { .. }) => {
|
|
// The suite does not exist: same outcome as before verification
|
|
// existed (probing callers simply try the next pocket/series)
|
|
return Err(format!("No Release file found for suite '{suite}' at {base_url}").into());
|
|
}
|
|
Ok(release::Verification::KeyringUnavailable { reason, .. }) => {
|
|
// An existing suite whose keys are unavailable must not be
|
|
// trusted, but this is not tampering: report and let callers
|
|
// move on
|
|
return Err(reason.into());
|
|
}
|
|
Err(e) => return Err(e.into()),
|
|
};
|
|
|
|
let components = verified.components();
|
|
if components.is_empty() {
|
|
return Err(format!("Components not found in the Release file of '{suite_url}'").into());
|
|
}
|
|
debug!("Found components: {:?}", components);
|
|
|
|
// Collect the failures of individual fetch attempts so that, if the
|
|
// package is not found, the final error explains what actually went
|
|
// wrong instead of misleadingly claiming a plain 'not found'
|
|
let mut fetch_errors: Vec<String> = Vec::new();
|
|
|
|
for component in components {
|
|
let url = crate::distro_info::get_sources_url(&base_url, series, pocket, &component);
|
|
|
|
debug!("Fetching sources from: {}", url);
|
|
|
|
let compressed_data = match fetch_index_bytes(&url).await {
|
|
Ok(data) => data,
|
|
Err(e) => {
|
|
debug!("Failed to fetch {url}: {e}");
|
|
fetch_errors.push(format!("{suite}/{component}: {e}"));
|
|
continue;
|
|
}
|
|
};
|
|
|
|
// The index must match the checksums listed in the signed Release
|
|
// file: this is what closes the 'substituted index with matching
|
|
// artifact checksums' man-in-the-middle attack
|
|
let suite_rel_path = format!("{component}/source/Sources.gz");
|
|
let compressed_data = match verified.verify_file(&suite_rel_path, &compressed_data) {
|
|
Ok(()) => compressed_data,
|
|
Err(verify_error) => {
|
|
// Busy mirrors and CDNs can serve an index generation
|
|
// slightly older or newer than the Release file fetched
|
|
// moments before. Pin the exact generation listed in the
|
|
// Release file via Debian's by-hash mechanism before
|
|
// failing.
|
|
match fetch_index_by_hash(&url, &verified, &suite_rel_path).await {
|
|
Ok(pinned) => {
|
|
debug!(
|
|
"index at '{url}' did not match the Release file: used the by-hash copy"
|
|
);
|
|
pinned
|
|
}
|
|
Err(by_hash_error) => {
|
|
debug!("by-hash fetch of '{}' failed: {}", url, by_hash_error);
|
|
return Err(release::VerifyError(format!(
|
|
"{verify_error}; the by-hash retry also failed: {by_hash_error}"
|
|
))
|
|
.into());
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
debug!(
|
|
"Downloaded Sources.gz for {}/{}/{}",
|
|
dist, series, component
|
|
);
|
|
|
|
if let Some(stanza) = parse_sources(&compressed_data, package_name, version)? {
|
|
if let Some(vcs) = &stanza.vcs_git
|
|
&& preferred_vcs.is_none()
|
|
{
|
|
preferred_vcs = Some(vcs.clone());
|
|
}
|
|
|
|
// If downloading from PPA, make sure we don't use a VCS
|
|
if from_ppa {
|
|
preferred_vcs = None;
|
|
}
|
|
|
|
let archive_url = format!("{base_url}/{0}", stanza.directory);
|
|
return Ok(PackageInfo {
|
|
dist,
|
|
series: series.to_string(),
|
|
stanza,
|
|
preferred_vcs,
|
|
archive_url,
|
|
});
|
|
}
|
|
}
|
|
|
|
let details = if fetch_errors.is_empty() {
|
|
String::new()
|
|
} else {
|
|
format!(" (last errors: {})", fetch_errors.join("; "))
|
|
};
|
|
Err(format!(
|
|
"Package '{}' not found in {}/{}{}",
|
|
package_name, dist, series, details
|
|
)
|
|
.into())
|
|
}
|
|
|
|
/// Try to find package information in a distribution, trying all series and pockets
|
|
async fn find_package(
|
|
package_name: &str,
|
|
dist: &str,
|
|
pocket: &str,
|
|
version: Option<&str>,
|
|
base_url: Option<&str>,
|
|
launchpad_vcs: Option<String>,
|
|
progress: ProgressCallback<'_>,
|
|
) -> Result<PackageInfo, Box<dyn Error>> {
|
|
let series_list = crate::distro_info::get_ordered_series_name(dist).await?;
|
|
|
|
// Collect the failures of the individual series/pocket probes so that,
|
|
// if nothing is found, the final error summarizes what went wrong
|
|
// (e.g. network errors, HTTP statuses) instead of a bare 'not found'
|
|
let mut attempt_errors: Vec<String> = Vec::new();
|
|
|
|
for (i, series) in series_list.iter().enumerate() {
|
|
if let Some(cb) = progress {
|
|
cb("", &format!("Checking {}...", series), i, series_list.len());
|
|
}
|
|
|
|
let pockets = if pocket.is_empty() {
|
|
crate::distro_info::get_dist_pockets(dist)?
|
|
} else {
|
|
vec![pocket.to_string()]
|
|
};
|
|
|
|
for p in pockets {
|
|
match get(
|
|
package_name,
|
|
series,
|
|
&p,
|
|
version,
|
|
base_url,
|
|
launchpad_vcs.clone(),
|
|
)
|
|
.await
|
|
{
|
|
Ok(info) => {
|
|
if i > 0 {
|
|
let location = if p.is_empty() {
|
|
format!("{dist}/{series}")
|
|
} else {
|
|
format!("{dist}/{series}-{p}")
|
|
};
|
|
warn!(
|
|
"Package '{}' not found in development release. Found in {}.",
|
|
package_name, location
|
|
);
|
|
} else {
|
|
let location = if p.is_empty() {
|
|
format!("{}/{}", dist, series)
|
|
} else {
|
|
format!("{}/{}-{}", dist, series, p)
|
|
};
|
|
debug!("Found package '{}' in {}", package_name, location);
|
|
}
|
|
return Ok(info);
|
|
}
|
|
Err(e) => {
|
|
// A Release verification failure is a security error,
|
|
// not a missing package: abort the search instead of
|
|
// silently probing other series/pockets
|
|
if e.downcast_ref::<release::VerifyError>().is_some() {
|
|
return Err(e);
|
|
}
|
|
// Remember the failure for the final error message, and
|
|
// keep probing the other series/pockets
|
|
let suite = if p.is_empty() {
|
|
series.clone()
|
|
} else {
|
|
format!("{series}-{p}")
|
|
};
|
|
attempt_errors.push(format!("{}: {}", suite, e));
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Keep only the last few attempts so the message stays readable
|
|
if attempt_errors.len() > 5 {
|
|
let drain_to = attempt_errors.len() - 5;
|
|
attempt_errors.drain(..drain_to);
|
|
}
|
|
let details = if attempt_errors.is_empty() {
|
|
String::new()
|
|
} else {
|
|
format!(" (last errors: {})", attempt_errors.join("; "))
|
|
};
|
|
|
|
Err(format!("Package '{}' not found.{}", package_name, details).into())
|
|
}
|
|
|
|
/// Fetch the 'Release' file at the root of a flat repository, and return its suite name
|
|
///
|
|
/// Flat repositories (e.g. 'https://pkg.example.org/deb/resolute/') serve the
|
|
/// Release file at their root, without any 'dists/' hierarchy. The suite is
|
|
/// read from the 'Codename' field, falling back to 'Suite'.
|
|
async fn get_flat_repo_series(base_url: &str) -> Result<String, Box<dyn Error>> {
|
|
let url = format!("{}/Release", base_url.trim_end_matches('/'));
|
|
let response = crate::distro_info::http_client().get(&url).send().await?;
|
|
if !response.status().is_success() {
|
|
return Err(format!(
|
|
"No Release file at '{}' (HTTP {}) - is '{}' the suite URL of a flat repository? \
|
|
External repositories must be passed as their full suite URL.",
|
|
url,
|
|
response.status(),
|
|
base_url
|
|
)
|
|
.into());
|
|
}
|
|
let content = response.text().await?;
|
|
|
|
for field in ["Codename", "Suite"] {
|
|
let prefix = format!("{field}:");
|
|
if let Some(line) = content.lines().find(|l| l.starts_with(&prefix)) {
|
|
return Ok(line[prefix.len()..].trim().to_string());
|
|
}
|
|
}
|
|
|
|
Err(format!("No Codename or Suite field in Release file at '{url}'").into())
|
|
}
|
|
|
|
/// Fetch an index pinned to the exact generation listed in the Release file
|
|
///
|
|
/// Uses Debian's by-hash mechanism, supported by the main archives (Debian,
|
|
/// Ubuntu, PPAs): the server serves the index generation matching the
|
|
/// Release file instead of whatever the mirror/CDN currently holds. The
|
|
/// returned content is verified again, so a by-hash-incapable mirror can
|
|
/// only cause a 404 here, never a silent mismatch.
|
|
async fn fetch_index_by_hash(
|
|
index_url: &str,
|
|
verified: &VerifiedRelease,
|
|
rel_path: &str,
|
|
) -> Result<Vec<u8>, Box<dyn Error>> {
|
|
let entry = verified.hash_for(rel_path).ok_or_else(|| {
|
|
format!("'{rel_path}' is not listed in the Release file: cannot fetch it by hash")
|
|
})?;
|
|
// By-hash URLs use the Release field names (MD5Sum, SHA1, SHA256, ...)
|
|
let algo = match entry.kind {
|
|
release::ChecksumKind::Md5 => "MD5Sum",
|
|
release::ChecksumKind::Sha1 => "SHA1",
|
|
release::ChecksumKind::Sha256 => "SHA256",
|
|
release::ChecksumKind::Sha512 => "SHA512",
|
|
};
|
|
// The by-hash layout replaces the file name with the hash reference:
|
|
// 'main/source/Sources.gz' is served at 'main/source/by-hash/SHA256/<digest>'
|
|
let (dir, _) = index_url
|
|
.rsplit_once('/')
|
|
.ok_or_else(|| format!("'{index_url}' has no parent directory"))?;
|
|
let url = format!("{dir}/by-hash/{algo}/{}", entry.hash);
|
|
|
|
let response = crate::distro_info::http_get_retried(&url).await?;
|
|
if !response.status().is_success() {
|
|
return Err(format!("HTTP {} for '{url}'", response.status()).into());
|
|
}
|
|
let data = response.bytes().await?.to_vec();
|
|
if data.is_empty() {
|
|
return Err(format!("empty body for '{url}'").into());
|
|
}
|
|
verified.verify_file(rel_path, &data)?;
|
|
Ok(data)
|
|
}
|
|
|
|
/// Fetch the compressed sources index at `url`
|
|
///
|
|
/// Transient transport failures and truncated bodies are retried; a
|
|
/// non-success HTTP status is a meaningful answer (e.g. a component that
|
|
/// does not exist in the suite) and is reported without retrying.
|
|
async fn fetch_index_bytes(url: &str) -> Result<Vec<u8>, String> {
|
|
const ATTEMPTS: u32 = 3;
|
|
let mut last_error = String::new();
|
|
for attempt in 1..=ATTEMPTS {
|
|
match crate::distro_info::http_get_retried(url).await {
|
|
Err(e) => {
|
|
last_error = e.to_string();
|
|
log::debug!("fetch of '{url}' failed (attempt {attempt}/{ATTEMPTS}): {last_error}");
|
|
tokio::time::sleep(std::time::Duration::from_millis(300 * u64::from(attempt)))
|
|
.await;
|
|
}
|
|
Ok(response) => {
|
|
if !response.status().is_success() {
|
|
return Err(format!("HTTP {}", response.status()));
|
|
}
|
|
match response.bytes().await {
|
|
Ok(bytes) if bytes.is_empty() => {
|
|
// CDNs occasionally answer 200 with an empty body
|
|
// under load: never a valid index, retry from scratch
|
|
last_error = "server returned an empty body".to_string();
|
|
log::debug!(
|
|
"empty body for '{url}' (attempt {attempt}/{ATTEMPTS}), retrying"
|
|
);
|
|
tokio::time::sleep(std::time::Duration::from_millis(
|
|
300 * u64::from(attempt),
|
|
))
|
|
.await;
|
|
}
|
|
Ok(bytes) => return Ok(bytes.to_vec()),
|
|
Err(e) => {
|
|
// Truncated or corrupted body: retry from scratch
|
|
last_error = e.to_string();
|
|
log::debug!(
|
|
"reading the body of '{url}' failed (attempt {attempt}/{ATTEMPTS}): {last_error}"
|
|
);
|
|
tokio::time::sleep(std::time::Duration::from_millis(
|
|
300 * u64::from(attempt),
|
|
))
|
|
.await;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Err(last_error)
|
|
}
|
|
|
|
/// Fetch the sources index of a flat repository
|
|
///
|
|
/// Flat repositories are free to serve any compressed variant of the index/// (or an uncompressed one), so try the usual candidates in turn. When the
|
|
/// repository published a Release file, each index candidate is
|
|
/// checksum-verified against it (failing on mismatch, since the artifact
|
|
/// hashes would come from the index itself).
|
|
async fn get_flat_repo_sources(
|
|
base_url: &str,
|
|
verified: Option<&VerifiedRelease>,
|
|
) -> Result<Vec<u8>, Box<dyn Error>> {
|
|
let base = base_url.trim_end_matches('/');
|
|
let mut errors = Vec::new();
|
|
for name in ["Sources.xz", "Sources.gz", "Sources"] {
|
|
let url = format!("{base}/{name}");
|
|
match crate::distro_info::http_get_retried(&url).await {
|
|
Ok(response) if response.status().is_success() => {
|
|
let data = response.bytes().await?.to_vec();
|
|
|
|
// Some flat Release files list their entries with a './' prefix
|
|
if let Some(release) = verified {
|
|
let mismatch = match release.verify_file(name, &data) {
|
|
Ok(()) => None,
|
|
Err(first) => release
|
|
.verify_file(&format!("./{name}"), &data)
|
|
.err()
|
|
.map(|_| first),
|
|
};
|
|
if let Some(e) = mismatch {
|
|
return Err(
|
|
format!("Verification of the repository index failed: {e}").into()
|
|
);
|
|
}
|
|
}
|
|
|
|
return Ok(data);
|
|
}
|
|
Ok(response) => errors.push(format!("{}: HTTP {}", url, response.status())),
|
|
Err(e) => errors.push(format!("{}: {}", url, e)),
|
|
}
|
|
}
|
|
|
|
Err(format!(
|
|
"Could not fetch the sources index from '{}': {}",
|
|
base,
|
|
errors.join(", ")
|
|
)
|
|
.into())
|
|
}
|
|
|
|
/// Lookup package information in an external flat repository
|
|
///
|
|
/// The URL points at the repository suite directory itself (e.g.
|
|
/// 'https://pkg.noctalia.dev/deb/resolute/'), as flat repositories have no
|
|
/// 'dists/<suite>' hierarchy. The suite name is read from the 'Release' file
|
|
/// unless overridden, and the package files are fetched relative to the URL
|
|
/// root, ignoring the 'Directory' field like apt does for flat repositories.
|
|
///
|
|
/// Packages from external repositories are always downloaded from the
|
|
/// repository itself; the 'Vcs-Git' of the stanza is never used, as it may
|
|
/// point to an arbitrary source.
|
|
///
|
|
/// Third-party flat repositories have no known signing key: when they
|
|
/// publish a Release file its checksums are enforced on the index (with a
|
|
/// hard error on mismatch), but the absence of a verifiable signature only
|
|
/// produces a warning, preserving the previous behavior for such repos.
|
|
pub async fn lookup_repository(
|
|
package: &str,
|
|
version: Option<&str>,
|
|
repo_url: &str,
|
|
series: Option<&str>,
|
|
progress: ProgressCallback<'_>,
|
|
) -> Result<PackageInfo, Box<dyn Error>> {
|
|
if let Some(cb) = progress {
|
|
cb(
|
|
&format!("Fetching repository info from {}...", repo_url),
|
|
"",
|
|
0,
|
|
0,
|
|
);
|
|
}
|
|
|
|
// Attempt to authenticate the repository. In non-strict mode only the
|
|
// absence of a Release file (or of a way to verify it) is tolerated:
|
|
// tampering evidence — an invalid or malformed signature — is a hard
|
|
// error even for third-party repositories.
|
|
let verified = match release::verify_suite(
|
|
repo_url.trim_end_matches('/'),
|
|
release::KeyringSource::None,
|
|
false,
|
|
)
|
|
.await
|
|
{
|
|
Ok(release::Verification::Available(verified)) => Some(verified),
|
|
Ok(release::Verification::Unavailable { .. }) => None,
|
|
Ok(release::Verification::KeyringUnavailable { reason, .. }) => {
|
|
warn!("Release verification of repository {repo_url} failed: {reason}");
|
|
None
|
|
}
|
|
Err(e) => return Err(e.into()),
|
|
};
|
|
|
|
// Prefer the suite name from the (fetched, possibly verified) Release
|
|
// file; fall back to a dedicated fetch when there is none
|
|
let resolved_series = if let Some(s) = series {
|
|
s.to_string()
|
|
} else if let Some(codename) = verified
|
|
.as_ref()
|
|
.and_then(|v| v.field("Codename").or_else(|| v.field("Suite")))
|
|
{
|
|
codename.to_string()
|
|
} else {
|
|
get_flat_repo_series(repo_url).await?
|
|
};
|
|
|
|
if let Some(cb) = progress {
|
|
cb(
|
|
&format!("Resolving package info for {}...", package),
|
|
"",
|
|
0,
|
|
0,
|
|
);
|
|
}
|
|
|
|
let sources = get_flat_repo_sources(repo_url, verified.as_ref()).await?;
|
|
let stanza = parse_sources(&sources, package, version)?
|
|
.ok_or_else(|| format!("Package '{package}' not found in repository {repo_url}"))?;
|
|
|
|
// The distribution is only used for naming (e.g. git branch layout);
|
|
// fall back to a neutral name for suites unknown to distro-info.
|
|
let dist = crate::distro_info::get_dist_from_series(&resolved_series)
|
|
.await
|
|
.unwrap_or_else(|_| "external".to_string());
|
|
|
|
Ok(PackageInfo {
|
|
dist,
|
|
series: resolved_series,
|
|
stanza,
|
|
preferred_vcs: None,
|
|
archive_url: repo_url.trim_end_matches('/').to_string(),
|
|
})
|
|
}
|
|
|
|
/// Lookup package information for a source package
|
|
///
|
|
/// This function obtains package information either directly from a specific series
|
|
/// or by searching across all series in a distribution.
|
|
///
|
|
/// # Arguments
|
|
/// * `package` - The name of the package to look up
|
|
/// * `version` - Optional specific version to look for
|
|
/// * `series` - Optional distribution series (e.g., "noble", "bookworm")
|
|
/// * `pocket` - Pocket to search in (e.g., "updates", "security", or "" for main)
|
|
/// * `dist` - Optional distribution name (e.g., "ubuntu", "debian")
|
|
/// * `base_url` - Optional base URL for the package archive (e.g., "https://ppa.launchpadcontent.net/user/ppa/ubuntu/")
|
|
/// * `repository` - Optional URL of an external flat repository suite (e.g., "https://pkg.noctalia.dev/deb/resolute/")
|
|
/// * `progress` - Optional progress callback
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub async fn lookup(
|
|
package: &str,
|
|
version: Option<&str>,
|
|
series: Option<&str>,
|
|
pocket: &str,
|
|
dist: Option<&str>,
|
|
base_url: Option<&str>,
|
|
repository: Option<&str>,
|
|
progress: ProgressCallback<'_>,
|
|
) -> Result<PackageInfo, Box<dyn Error>> {
|
|
// External flat repository: the URL is the suite root, distribution
|
|
// resolution and pocket logic do not apply
|
|
if let Some(repo_url) = repository {
|
|
return lookup_repository(package, version, repo_url, series, progress).await;
|
|
}
|
|
|
|
// Resolve the distribution early so we can check Launchpad once
|
|
let resolved_dist = dist.unwrap_or_else(||
|
|
// Use auto-detection to see if current distro is ubuntu, or fallback to debian by default
|
|
if std::process::Command::new("lsb_release").arg("-i").arg("-s").output()
|
|
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_lowercase()).unwrap_or_default() == "ubuntu" {
|
|
"ubuntu"
|
|
} else {
|
|
"debian"
|
|
}
|
|
);
|
|
|
|
// Check Launchpad repository once for Ubuntu packages, rather than
|
|
// once per series/pocket combination inside get().
|
|
let launchpad_vcs = if resolved_dist == "ubuntu" && base_url.is_none() {
|
|
if let Some(cb) = progress {
|
|
cb("", &format!("Checking Launchpad for {}...", package), 0, 0);
|
|
}
|
|
match check_launchpad_repo(package).await {
|
|
Ok(Some(lp_url)) => {
|
|
debug!("Found Launchpad URL: {}", lp_url);
|
|
Some(lp_url)
|
|
}
|
|
Ok(None) => None,
|
|
Err(e) => {
|
|
debug!(
|
|
"Launchpad check failed (will use stanza Vcs-Git if available): {}",
|
|
e
|
|
);
|
|
None
|
|
}
|
|
}
|
|
} else {
|
|
None
|
|
};
|
|
|
|
// Obtain the package information, either directly in a series or with a search in all series
|
|
let package_info = if let Some(s) = series {
|
|
if let Some(cb) = progress {
|
|
cb(
|
|
&format!("Resolving package info for {}...", package),
|
|
"",
|
|
0,
|
|
0,
|
|
);
|
|
}
|
|
|
|
// Get the package information from that series and pocket
|
|
get(package, s, pocket, version, base_url, launchpad_vcs).await?
|
|
} else {
|
|
if let Some(cb) = progress {
|
|
cb(
|
|
&format!(
|
|
"Searching for package {} in {}...",
|
|
package,
|
|
if base_url.is_none() {
|
|
resolved_dist
|
|
} else {
|
|
"ppa"
|
|
}
|
|
),
|
|
"",
|
|
0,
|
|
0,
|
|
);
|
|
}
|
|
|
|
// Try to find the package in all series from that dist
|
|
find_package(
|
|
package,
|
|
resolved_dist,
|
|
pocket,
|
|
version,
|
|
base_url,
|
|
launchpad_vcs,
|
|
progress,
|
|
)
|
|
.await?
|
|
};
|
|
|
|
Ok(package_info)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
/// `user/ppa_name` splits into its two parts.
|
|
#[test]
|
|
fn split_ppa_parses_the_canonical_form() {
|
|
assert_eq!(split_ppa("user/my-ppa"), Ok(("user", "my-ppa")));
|
|
}
|
|
|
|
/// Anything but exactly two non-empty segments is rejected, with the
|
|
/// canonical message.
|
|
#[test]
|
|
fn split_ppa_rejects_malformed_references() {
|
|
for bad in ["", "user", "user/", "/ppa", "a/b/c"] {
|
|
let err = split_ppa(bad).unwrap_err();
|
|
assert!(err.contains("Invalid PPA format"), "{err}");
|
|
assert!(err.contains(bad), "{err}");
|
|
}
|
|
}
|
|
|
|
use super::*;
|
|
|
|
/// Serve canned byte responses on a local port, one per connection (the
|
|
/// last response repeats), and return the base URL
|
|
///
|
|
/// The canned responses must use 'Connection: close' so the client opens
|
|
/// a fresh connection (and receives a fresh response) per request.
|
|
fn serve_responses(responses: Vec<Vec<u8>>) -> String {
|
|
use std::io::{Read, Write};
|
|
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
|
let addr = listener.local_addr().unwrap();
|
|
std::thread::spawn(move || {
|
|
for (served, stream) in listener.incoming().flatten().enumerate() {
|
|
let index = served.min(responses.len() - 1);
|
|
let mut stream = stream;
|
|
// Drain the request first: closing with unread inbound data
|
|
// would send a TCP RST and destroy the response in flight
|
|
let mut buf = [0u8; 4096];
|
|
loop {
|
|
match stream.read(&mut buf) {
|
|
Ok(0) => break,
|
|
Ok(n) if buf[..n].windows(4).any(|w| w == b"\r\n\r\n") => break,
|
|
Ok(_) => continue,
|
|
Err(_) => break,
|
|
}
|
|
}
|
|
let _ = stream.write_all(&responses[index]);
|
|
let _ = stream.flush();
|
|
}
|
|
});
|
|
format!("http://{addr}/Sources.gz")
|
|
}
|
|
|
|
/// A CDN answering 200 with an empty body (observed under load) must be
|
|
/// retried instead of failing the index checksum verification
|
|
#[tokio::test]
|
|
async fn fetch_index_bytes_retries_empty_body() {
|
|
let valid = b"Package: hello\nVersion: 1.0\n\n";
|
|
let empty_response =
|
|
b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec();
|
|
let mut valid_response = format!(
|
|
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
|
valid.len()
|
|
)
|
|
.into_bytes();
|
|
valid_response.extend_from_slice(valid);
|
|
|
|
let url = serve_responses(vec![empty_response, valid_response]);
|
|
let data = fetch_index_bytes(&url).await.unwrap();
|
|
assert_eq!(data, valid);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_check_launchpad_repo() {
|
|
// "hello" should exist on Launchpad for Ubuntu
|
|
let url = check_launchpad_repo("hello").await.unwrap();
|
|
assert!(url.is_some());
|
|
assert_eq!(
|
|
url.unwrap(),
|
|
"https://git.launchpad.net/ubuntu/+source/hello"
|
|
);
|
|
|
|
// "this-package-should-not-exist-12345" should not exist
|
|
let url = check_launchpad_repo("this-package-should-not-exist-12345")
|
|
.await
|
|
.unwrap();
|
|
assert!(url.is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn test_parse_sources() {
|
|
use flate2::Compression;
|
|
use flate2::write::GzEncoder;
|
|
use std::io::Write;
|
|
|
|
let data = "Package: hello
|
|
Version: 2.10-2
|
|
Format: 3.0 (quilt)
|
|
Directory: pool/main/h/hello
|
|
Vcs-Git: https://salsa.debian.org/debian/hello.git
|
|
|
|
Package: other
|
|
Version: 1.0
|
|
";
|
|
|
|
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
|
encoder.write_all(data.as_bytes()).unwrap();
|
|
let compressed = encoder.finish().unwrap();
|
|
|
|
let info = parse_sources(&compressed, "hello", None).unwrap().unwrap();
|
|
assert_eq!(info.package, "hello");
|
|
assert_eq!(info.version, "2.10-2");
|
|
assert_eq!(info.format, "3.0 (quilt)");
|
|
assert_eq!(info.directory, "pool/main/h/hello");
|
|
assert_eq!(
|
|
info.vcs_git.unwrap(),
|
|
"https://salsa.debian.org/debian/hello.git"
|
|
);
|
|
|
|
let none = parse_sources(&compressed, "missing", None).unwrap();
|
|
assert!(none.is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn test_parse_sources_compression_variants() {
|
|
use std::io::Write;
|
|
use xz2::write::XzEncoder;
|
|
|
|
let data = "Package: hello
|
|
Version: 1.0
|
|
Directory: pool/main/h/hello
|
|
";
|
|
|
|
// xz-compressed index (e.g. the 'Sources.xz' of flat repositories)
|
|
let mut encoder = XzEncoder::new(Vec::new(), 6);
|
|
encoder.write_all(data.as_bytes()).unwrap();
|
|
let compressed = encoder.finish().unwrap();
|
|
let info = parse_sources(&compressed, "hello", None).unwrap().unwrap();
|
|
assert_eq!(info.version, "1.0");
|
|
|
|
// Uncompressed index
|
|
let info = parse_sources(data.as_bytes(), "hello", None)
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(info.version, "1.0");
|
|
}
|
|
|
|
#[test]
|
|
fn test_parse_sources_many_blank_stanzas() {
|
|
// A crafted index with many consecutive package-less stanzas must be
|
|
// iterated without recursion: 100k blank stanzas would overflow the
|
|
// stack with the old recursive 'return self.next()' implementation
|
|
let mut data = String::new();
|
|
for _ in 0..100_000 {
|
|
data.push_str("Not-Really-Package: x\n\n");
|
|
}
|
|
data.push_str("Package: hello\nVersion: 1.0\n");
|
|
|
|
let info = parse_sources(data.as_bytes(), "hello", None)
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(info.package, "hello");
|
|
assert_eq!(info.version, "1.0");
|
|
}
|
|
|
|
#[test]
|
|
fn test_parse_sources_stanza_without_version() {
|
|
// A stanza with a 'Package' but no 'Version' field is malformed
|
|
// remote data: it must be skipped rather than panic
|
|
let data = "Package: noversion\nDirectory: pool/main/n/noversion\n\n\
|
|
Package: hello\nVersion: 1.0\n";
|
|
|
|
let info = parse_sources(data.as_bytes(), "hello", None)
|
|
.unwrap()
|
|
.unwrap();
|
|
assert_eq!(info.package, "hello");
|
|
assert_eq!(info.version, "1.0");
|
|
|
|
assert!(
|
|
parse_sources(data.as_bytes(), "noversion", None)
|
|
.unwrap()
|
|
.is_none()
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_find_package_fallback() {
|
|
// python2.7 is in bullseye but not above
|
|
let info = find_package("python2.7", "debian", "", None, None, None, None)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(info.stanza.package, "python2.7");
|
|
assert_eq!(info.series, "bullseye")
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_find_package_devel() {
|
|
// hello is in sid
|
|
let info = find_package("hello", "debian", "", None, None, None, None)
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(info.stanza.package, "hello");
|
|
assert_eq!(info.series, "sid")
|
|
}
|
|
}
|