net: retry flaky archive fetches and pin index downloads via by-hash

Busy mirrors and CDNs routinely break bulk fetches: pooled keep-alive
connections get closed remotely ('error sending request'), downloads are
cut short (surfacing as bogus checksum mismatches), and index generations
momentarily drift from the Release file fetched moments before.

- shared client: short idle-pool timeout and TCP keepalive, and a
  bounded-retry GET helper now used for index, Release, keyring and
  Launchpad fetches (previously reqwest::get, which has no timeouts)
- downloads: retry the whole download, and check the content length so
  truncation is reported as such instead of a checksum mismatch
- sources index: on a checksum mismatch against the Release file, retry
  pinned to the exact listed generation via Debian's by-hash mechanism;
  body-read errors are retried and reported per component instead of
  aborting the whole lookup

src/package_info.rs

@@ -387,30 +387,41 @@ async fn get(
 
         debug!("Fetching sources from: {}", url);
 
-        let response = match crate::distro_info::http_client().get(&url).send().await {
-            Ok(resp) => resp,
+        let compressed_data = match fetch_index_bytes(&url).await {
+            Ok(data) => data,
             Err(e) => {
-                debug!("Failed to fetch {}: {}", url, e);
-                fetch_errors.push(format!("{suite}/{component}: {}", e));
+                debug!("Failed to fetch {url}: {e}");
+                fetch_errors.push(format!("{suite}/{component}: {e}"));
                 continue;
             }
         };
 
-        if !response.status().is_success() {
-            debug!("Failed to fetch {}: status {}", url, response.status());
-            fetch_errors.push(format!("{suite}/{component}: HTTP {}", response.status()));
-            continue;
-        }
-
-        let compressed_data = response.bytes().await?;
-
         // The index must match the checksums listed in the signed Release
         // file: this is what closes the 'substituted index with matching
         // artifact checksums' man-in-the-middle attack
         let suite_rel_path = format!("{component}/source/Sources.gz");
-        verified
-            .verify_file(&suite_rel_path, &compressed_data)
-            .map_err(release::VerifyError)?;
+        let compressed_data = match verified.verify_file(&suite_rel_path, &compressed_data) {
+            Ok(()) => compressed_data,
+            Err(verify_error) => {
+                // Busy mirrors and CDNs can serve an index generation
+                // slightly older or newer than the Release file fetched
+                // moments before. Pin the exact generation listed in the
+                // Release file via Debian's by-hash mechanism before
+                // failing.
+                match fetch_index_by_hash(&url, &verified, &suite_rel_path).await {
+                    Ok(pinned) => {
+                        debug!(
+                            "index at '{url}' did not match the Release file: used the by-hash copy"
+                        );
+                        pinned
+                    }
+                    Err(by_hash_error) => {
+                        debug!("by-hash fetch of '{}' failed: {}", url, by_hash_error);
+                        return Err(release::VerifyError(verify_error).into());
+                    }
+                }
+            }
+        };
 
         debug!(
             "Downloaded Sources.gz for {}/{}/{}",
@@ -577,10 +588,86 @@ async fn get_flat_repo_series(base_url: &str) -> Result<String, Box<dyn Error>>
     Err(format!("No Codename or Suite field in Release file at '{url}'").into())
 }
 
+/// Fetch an index pinned to the exact generation listed in the Release file
+///
+/// Uses Debian's by-hash mechanism, supported by the main archives (Debian,
+/// Ubuntu, PPAs): the server serves the index generation matching the
+/// Release file instead of whatever the mirror/CDN currently holds. The
+/// returned content is verified again, so a by-hash-incapable mirror can
+/// only cause a 404 here, never a silent mismatch.
+async fn fetch_index_by_hash(
+    index_url: &str,
+    verified: &VerifiedRelease,
+    rel_path: &str,
+) -> Result<Vec<u8>, Box<dyn Error>> {
+    let entry = verified.hash_for(rel_path).ok_or_else(|| {
+        format!("'{rel_path}' is not listed in the Release file: cannot fetch it by hash")
+    })?;
+    // By-hash URLs use the Release field names (MD5Sum, SHA1, SHA256, ...)
+    let algo = match entry.kind {
+        release::ChecksumKind::Md5 => "MD5Sum",
+        release::ChecksumKind::Sha1 => "SHA1",
+        release::ChecksumKind::Sha256 => "SHA256",
+        release::ChecksumKind::Sha512 => "SHA512",
+    };
+    // The by-hash layout replaces the file name with the hash reference:
+    // 'main/source/Sources.gz' is served at 'main/source/by-hash/SHA256/<digest>'
+    let (dir, _) = index_url
+        .rsplit_once('/')
+        .ok_or_else(|| format!("'{index_url}' has no parent directory"))?;
+    let url = format!("{dir}/by-hash/{algo}/{}", entry.hash);
+
+    let response = crate::distro_info::http_get_retried(&url).await?;
+    if !response.status().is_success() {
+        return Err(format!("HTTP {} for '{url}'", response.status()).into());
+    }
+    let data = response.bytes().await?.to_vec();
+    verified.verify_file(rel_path, &data)?;
+    Ok(data)
+}
+
+/// Fetch the compressed sources index at `url`
+///
+/// Transient transport failures and truncated bodies are retried; a
+/// non-success HTTP status is a meaningful answer (e.g. a component that
+/// does not exist in the suite) and is reported without retrying.
+async fn fetch_index_bytes(url: &str) -> Result<Vec<u8>, String> {
+    const ATTEMPTS: u32 = 3;
+    let mut last_error = String::new();
+    for attempt in 1..=ATTEMPTS {
+        match crate::distro_info::http_get_retried(url).await {
+            Err(e) => {
+                last_error = e.to_string();
+                log::debug!("fetch of '{url}' failed (attempt {attempt}/{ATTEMPTS}): {last_error}");
+                tokio::time::sleep(std::time::Duration::from_millis(300 * u64::from(attempt))).await;
+            }
+            Ok(response) => {
+                if !response.status().is_success() {
+                    return Err(format!("HTTP {}", response.status()));
+                }
+                match response.bytes().await {
+                    Ok(bytes) => return Ok(bytes.to_vec()),
+                    Err(e) => {
+                        // Truncated or corrupted body: retry from scratch
+                        last_error = e.to_string();
+                        log::debug!(
+                            "reading the body of '{url}' failed (attempt {attempt}/{ATTEMPTS}): {last_error}"
+                        );
+                        tokio::time::sleep(
+                            std::time::Duration::from_millis(300 * u64::from(attempt)),
+                        )
+                        .await;
+                    }
+                }
+            }
+        }
+    }
+    Err(last_error)
+}
+
 /// Fetch the sources index of a flat repository
 ///
-/// Flat repositories are free to serve any compressed variant of the index
-/// (or an uncompressed one), so try the usual candidates in turn. When the
+/// Flat repositories are free to serve any compressed variant of the index/// (or an uncompressed one), so try the usual candidates in turn. When the
 /// repository published a Release file, each index candidate is
 /// checksum-verified against it (failing on mismatch, since the artifact
 /// hashes would come from the index itself).
@@ -592,7 +679,7 @@ async fn get_flat_repo_sources(
     let mut errors = Vec::new();
     for name in ["Sources.xz", "Sources.gz", "Sources"] {
         let url = format!("{base}/{name}");
-        match crate::distro_info::http_client().get(&url).send().await {
+        match crate::distro_info::http_get_retried(&url).await {
             Ok(response) if response.status().is_success() => {
                 let data = response.bytes().await?.to_vec();