net: retry flaky archive fetches and pin index downloads via by-hash

Busy mirrors and CDNs routinely break bulk fetches: pooled keep-alive
connections get closed remotely ('error sending request'), downloads are
cut short (surfacing as bogus checksum mismatches), and index generations
momentarily drift from the Release file fetched moments before.

- shared client: short idle-pool timeout and TCP keepalive, and a
  bounded-retry GET helper now used for index, Release, keyring and
  Launchpad fetches (previously reqwest::get, which has no timeouts)
- downloads: retry the whole download, and check the content length so
  truncation is reported as such instead of a checksum mismatch
- sources index: on a checksum mismatch against the Release file, retry
  pinned to the exact listed generation via Debian's by-hash mechanism;
  body-read errors are retried and reported per component instead of
  aborting the whole lookup

src/distro_info.rs

@@ -53,9 +53,14 @@ lazy_static! {
 
     // Shared HTTP client used for all outgoing plain requests: timeouts keep
     // a hanging remote (connect or transfer) from stalling pkh indefinitely.
+    // The short pool idle timeout and TCP keepalive avoid reusing keep-alive
+    // connections that the remote closed in the meantime, which surfaces as
+    // spurious 'error sending request' failures on busy mirrors/CDNs.
     static ref HTTP_CLIENT: reqwest::Client = reqwest::Client::builder()
         .connect_timeout(Duration::from_secs(10))
         .timeout(Duration::from_secs(30))
+        .pool_idle_timeout(Duration::from_secs(10))
+        .tcp_keepalive(Duration::from_secs(30))
         .build()
         .expect("building the shared HTTP client with static options cannot fail");
 }
@@ -66,6 +71,49 @@ pub(crate) fn http_client() -> &'static reqwest::Client {
     &HTTP_CLIENT
 }
 
+/// GET `url` with bounded retries on transient transport errors (a pooled
+/// keep-alive connection closed by the remote, a momentary network hiccup,
+/// ...): these always succeed again on a fresh connection, and mirrors are
+/// busy enough that unguarded single attempts make bulk operations flaky.
+///
+/// The response status is not inspected: 404s and the like are meaningful
+/// answers, not transport failures.
+pub(crate) async fn http_get_retried(url: &str) -> reqwest::Result<reqwest::Response> {
+    http_get_retried_with_timeout(url, None).await
+}
+
+/// [`http_get_retried`] with a per-request timeout override, for large
+/// streaming downloads that exceed the shared client's total timeout
+pub(crate) async fn http_get_retried_with_timeout(
+    url: &str,
+    timeout: Option<Duration>,
+) -> reqwest::Result<reqwest::Response> {
+    const ATTEMPTS: u32 = 3;
+    let mut last_error: Option<reqwest::Error> = None;
+    for attempt in 0..ATTEMPTS {
+        let mut request = http_client().get(url);
+        if let Some(timeout) = timeout {
+            request = request.timeout(timeout);
+        }
+        match request.send().await {
+            Ok(response) => return Ok(response),
+            Err(e) => {
+                if attempt + 1 < ATTEMPTS {
+                    log::debug!(
+                        "GET '{url}' failed (attempt {}/{}, retrying): {}",
+                        attempt + 1,
+                        ATTEMPTS,
+                        e
+                    );
+                    tokio::time::sleep(Duration::from_millis(300 * (u64::from(attempt) + 1))).await;
+                }
+                last_error = Some(e);
+            }
+        }
+    }
+    Err(last_error.expect("at least one attempt was made"))
+}
+
 /// Parse an optional '%Y-%m-%d' date from a CSV cell, warning instead of
 /// panicking on invalid remote data
 fn parse_optional_date(value: Option<&str>, series: &str, field: &str) -> Option<NaiveDate> {