net: retry flaky archive fetches and pin index downloads via by-hash

Busy mirrors and CDNs routinely break bulk fetches: pooled keep-alive
connections get closed remotely ('error sending request'), downloads are
cut short (surfacing as bogus checksum mismatches), and index generations
momentarily drift from the Release file fetched moments before.

- shared client: short idle-pool timeout and TCP keepalive, and a
  bounded-retry GET helper now used for index, Release, keyring and
  Launchpad fetches (previously reqwest::get, which has no timeouts)
- downloads: retry the whole download, and check the content length so
  truncation is reported as such instead of a checksum mismatch
- sources index: on a checksum mismatch against the Release file, retry
  pinned to the exact listed generation via Debian's by-hash mechanism;
  body-read errors are retried and reported per component instead of
  aborting the whole lookup

src/pull.rs

@@ -309,23 +309,53 @@ async fn download_file_checksum(
     algo: crate::package_info::ChecksumAlgo,
     target_dir: &Path,
     progress: ProgressCallback<'_>,
+) -> Result<(), Box<dyn Error>> {
+    // Archive mirrors and CDNs are busy enough that single attempts fail
+    // spuriously (dropped connections, truncated bodies, index generations
+    // momentarily out of sync): retry the whole download a few times before
+    // reporting the last failure.
+    const ATTEMPTS: u32 = 3;
+    let mut last_error: Box<dyn Error> = String::new().into();
+    for attempt in 1..=ATTEMPTS {
+        match download_file_checksum_once(url, checksum, algo, target_dir, progress).await {
+            Ok(()) => return Ok(()),
+            Err(e) => {
+                log::warn!(
+                    "download of '{url}' failed (attempt {attempt}/{ATTEMPTS}): {e}"
+                );
+                last_error = e;
+                tokio::time::sleep(std::time::Duration::from_millis(500 * u64::from(attempt))).await;
+            }
+        }
+    }
+    Err(format!(
+        "downloading '{url}' failed after {ATTEMPTS} attempts: {last_error}"
+    )
+    .into())
+}
+
+/// One download attempt of [`download_file_checksum`], verifying the
+/// content length and the expected checksum
+async fn download_file_checksum_once(
+    url: &str,
+    checksum: &str,
+    algo: crate::package_info::ChecksumAlgo,
+    target_dir: &Path,
+    progress: ProgressCallback<'_>,
 ) -> Result<(), Box<dyn Error>> {
     // Download with the shared client (connect timeout). Large orig tarballs
     // can legitimately take longer than the client's default total timeout,
     // so use a generous per-request timeout for streaming downloads
-    let response = crate::distro_info::http_client()
-        .get(url)
-        .timeout(std::time::Duration::from_secs(30 * 60))
-        .send()
-        .await?;
+    let response = crate::distro_info::http_get_retried_with_timeout(
+        url,
+        Some(std::time::Duration::from_secs(30 * 60)),
+    )
+    .await
+    .map_err(|e| Box::new(e) as Box<dyn Error>)?;
     if !response.status().is_success() {
         return Err(format!("Failed to download '{}' : {}", url, response.status()).into());
     }
-
-    let total_size = response
-        .content_length()
-        .ok_or(format!("Failed to get content length from '{}'", url))?;
-    let mut index = 0;
+    let total_size = response.content_length();
 
     // Target file: extract file name from URL
     let filename = Path::new(url)
@@ -341,18 +371,30 @@ async fn download_file_checksum(
     let mut stream = response.bytes_stream();
     // Accumulate the downloaded bytes so we can compute the final digest with the
     // correct algorithm once the download is complete.
-    let mut buffer: Vec<u8> = Vec::with_capacity(total_size as usize);
+    let mut buffer: Vec<u8> = Vec::with_capacity(total_size.unwrap_or(0) as usize);
     while let Some(item) = stream.next().await {
         let chunk = item?;
         file.write_all(&chunk)?;
         buffer.extend_from_slice(&chunk);
 
-        if let Some(cb) = progress {
-            index = min(index + chunk.len(), total_size as usize);
-            cb("", "Downloading...", index, total_size as usize);
+        if let (Some(cb), Some(total)) = (progress, total_size) {
+            let index = min(buffer.len(), total as usize);
+            cb("", "Downloading...", index, total as usize);
         }
     }
 
+    // A dropped connection can end the stream early: never hand a truncated
+    // file to the checksum check (its mismatch message would hide the cause)
+    if let Some(total) = total_size
+        && buffer.len() != total as usize
+    {
+        return Err(format!(
+            "incomplete download from '{url}': got {} of {total} bytes",
+            buffer.len()
+        )
+        .into());
+    }
+
     // Verify checksum using the algorithm specified for this file
     let calculated_checksum = algo.hex_digest(&buffer);
     if calculated_checksum != checksum {