vhaudiquet/pkh
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better logging

Browse Source 
Use callbacks to report progress, indicatif for progress bars, no direct logs but progress messages
This commit is contained in:
Valentin Haudiquet
2025-11-27 19:34:46 +01:00
parent a4d2441b0a
commit 311304666f
5 changed files with 158 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Cargo.toml
View File

@@ -10,14 +10,18 @@ cmd_lib = "2.0.0"
flate2 = "1.1.5"
serde = { version = "1.0.228", features = ["derive"] }
csv = "1.3.0"
reqwest = { version = "0.12.9", features = ["blocking", "json"] }
reqwest = { version = "0.12.9", features = ["blocking", "json", "stream"] }
git2 = "0.19.0"
regex = "1"
chrono = "0.4"
tokio = { version = "1.41.1", features = ["full"] }
sha2 = "0.10.8"
hex = "0.4.3"
tracing = "0.1.42"
log = "0.4.28"
indicatif = "0.17"
indicatif-log-bridge = "0.2.3"
env_logger = "0.11.8"
futures-util = { version = "0.3.31", features = ["tokio-io"] }
[dev-dependencies]
tempfile = "3.10.1"

132
src/get.rs
View File

@@ -1,69 +1,117 @@
use serde::Deserialize;
use std::error::Error;
use std::path::Path;
use std::cmp::min;
use pkh::package_info;
use pkh::package_info::PackageInfo;
use std::process::Command;
fn clone_repo(url: &str, package: &str, cwd: Option<&Path>) -> Result<(), Box<dyn Error>> {
println!("Cloning {} into {}...", url, package);
use log::{debug};
use regex::Regex;
use pkh::ProgressCallback;
fn clone_repo(url: &str, package: &str, cwd: Option<&Path>, progress: ProgressCallback<'_>) -> Result<(), Box<dyn Error>> {
let target_path = if let Some(path) = cwd {
path.join(package)
} else {
Path::new(package).to_path_buf()
};
let mut callbacks = git2::RemoteCallbacks::new();
if let Some(ref progress_cb) = progress {
callbacks.transfer_progress(move |stats| {
(progress_cb)("", "Receiving objects...", stats.received_objects(), stats.total_objects());
true
});
callbacks.sideband_progress(move |data| {
let msg = String::from_utf8_lossy(data);
let re = Regex::new(r"(.*):[ ]*([0-9]*)% \(([0-9]*)/([0-9]*)\)").unwrap();
if let Some(caps) = re.captures(msg.trim()) {
let msg = caps.get(1).map_or("", |m| m.as_str()).to_string();
let objects = caps.get(3).map_or("", |m| m.as_str()).to_string().parse::<usize>().unwrap_or(0);
let total = caps.get(4).map_or("", |m| m.as_str()).to_string().parse::<usize>().unwrap_or(0);
git2::Repository::clone(url, target_path)?;
Ok(())
(progress_cb)("", msg.as_str(), objects, total);
}
true
});
}
let mut fetch_options = git2::FetchOptions::new();
fetch_options.remote_callbacks(callbacks);
let mut builder = git2::build::RepoBuilder::new();
builder.fetch_options(fetch_options);
return match builder.clone(url, &target_path) {
Ok(_repo) => {
Ok(())
}
Err(e) => {
Err(format!("Failed to clone: {}", e).into())
}
};
}
use sha2::{Sha256, Digest};
use std::fs::File;
use std::io::Write;
use futures_util::StreamExt;
fn checkout_pristine_tar(package_dir: &Path, filename: &str) -> Result<(), Box<dyn Error>> {
println!("Attempting to checkout {} using pristine-tar...", filename);
let status = Command::new("pristine-tar")
let output = Command::new("pristine-tar")
.current_dir(package_dir)
.args(&["checkout", format!("../{filename}").as_str()])
.status()?;
.output()
.expect("pristine-tar checkout failed");
if !status.success() {
return Err(format!("pristine-tar checkout failed with status: {}", status).into());
if !output.status.success() {
return Err(format!("pristine-tar checkout failed with status: {}", output.status).into());
}
Ok(())
}
async fn download_file_checksum(url: &str, checksum: &str, target_dir: &Path) -> Result<(), Box<dyn Error>> {
async fn download_file_checksum(url: &str, checksum: &str, target_dir: &Path, progress: ProgressCallback<'_>) -> Result<(), Box<dyn Error>> {
// Download with reqwest
let response = reqwest::get(url).await?;
if !response.status().is_success() {
return Err(format!("Failed to download: {}", response.status()).into());
return Err(format!("Failed to download '{}' : {}", &url, response.status()).into());
}
let total_size = response.content_length().ok_or(format!("Failed to get content length from '{}'", &url))?;
let mut index = 0;
let content = response.bytes().await?;
// Target file: extract file name from URL
let filename = Path::new(url).file_name().unwrap().to_str().unwrap();
let path = target_dir.join(filename);
let mut file = File::create(path)?;
// Verify checksum
// Download chunk by chunk to disk, while updating hasher for checksum
let mut stream = response.bytes_stream();
let mut hasher = Sha256::new();
hasher.update(&content);
while let Some(item) = stream.next().await {
let chunk = item?;
file.write_all(&chunk)?;
hasher.update(&chunk);
if let Some(cb) = progress {
index = min(index + &chunk.len(), total_size as usize);
cb("", "Downloading...", index, total_size as usize);
}
}
// Verify checksum
let result = hasher.finalize();
let calculated_checksum = hex::encode(result);
if calculated_checksum != checksum {
return Err(format!("Checksum mismatch! Expected {}, got {}", checksum, calculated_checksum).into());
}
// Extract file name from URL
let filename = Path::new(url).file_name().unwrap().to_str().unwrap();
// Write to disk
let path = target_dir.join(filename);
let mut file = File::create(path)?;
file.write_all(&content)?;
Ok(())
}
@@ -81,7 +129,7 @@ fn setup_pristine_tar_branch(package_dir: &Path, dist: &str) -> Result<(), Box<d
let (branch, _) = branch_result?;
if let Some(name) = branch.name()? {
if name.ends_with(&format!("/{dist}/pristine-tar")) {
println!("Found remote pristine-tar branch: {}", name);
debug!("Found remote pristine-tar branch: {}", name);
let commit = branch.get().peel_to_commit()?;
@@ -91,17 +139,17 @@ fn setup_pristine_tar_branch(package_dir: &Path, dist: &str) -> Result<(), Box<d
// Set upstream
local_branch.set_upstream(Some(name))?;
println!("Created local pristine-tar branch tracking {}", name);
debug!("Created local pristine-tar branch tracking {}", name);
return Ok(());
}
}
}
println!("No remote pristine-tar branch found.");
debug!("No remote pristine-tar branch found.");
Ok(())
}
async fn fetch_orig_tarball(info: &PackageInfo, cwd: Option<&Path>) -> Result<(), Box<dyn Error>> {
async fn fetch_orig_tarball(info: &PackageInfo, cwd: Option<&Path>, progress: ProgressCallback<'_>) -> Result<(), Box<dyn Error>> {
let package_dir = if let Some(path) = cwd {
path.join(&info.stanza.package)
} else {
@@ -118,22 +166,25 @@ async fn fetch_orig_tarball(info: &PackageInfo, cwd: Option<&Path>) -> Result<()
// 1. Try executing pristine-tar
// Setup pristine-tar branch if needed (by tracking remote branch)
setup_pristine_tar_branch(&package_dir, info.dist.as_str());
let _ = setup_pristine_tar_branch(&package_dir, info.dist.as_str());
if let Err(e) = checkout_pristine_tar(&package_dir, filename.as_str()) {
println!("pristine-tar failed: {}. Falling back to archive download.", e);
debug!("pristine-tar failed: {}. Falling back to archive download.", e);
// 2. Fallback to archive download
// We download to the parent directory of the package repo (which is standard for build tools)
// or the current directory if cwd is None (which effectively is the parent of the package dir)
let target_dir = cwd.unwrap_or_else(|| Path::new("."));
download_file_checksum(format!("{}/{}", &info.archive_url, filename).as_str(), &orig_file.sha256, target_dir).await?;
download_file_checksum(format!("{}/{}", &info.archive_url, filename).as_str(), &orig_file.sha256, target_dir, progress).await?;
}
Ok(())
}
pub async fn get(package: &str, _version: &str, series: &str, pocket: &str, _ppa: &str, cwd: Option<&Path>) -> Result<(), Box<dyn Error>> {
pub async fn get(package: &str, _version: &str, series: &str, pocket: &str, _ppa: &str, cwd: Option<&Path>, progress: ProgressCallback<'_>) -> Result<(), Box<dyn Error>> {
if let Some(cb) = progress {
cb(&format!("Resolving package info for {}...", package), "", 0, 0);
}
let package_info = package_info::get(package, series, pocket).await;
let package_dir = if let Some(path) = cwd {
@@ -144,11 +195,18 @@ pub async fn get(package: &str, _version: &str, series: &str, pocket: &str, _ppa
if let Ok(Some(info)) = package_info {
if let Some(ref url) = info.preferred_vcs {
clone_repo(url.as_str(), package, Some(&package_dir))?;
fetch_orig_tarball(&info, Some(&package_dir)).await?;
if let Some(cb) = progress {
cb(&format!("Cloning {}...", url), "", 0, 0);
}
clone_repo(url.as_str(), package, Some(&package_dir), progress)?;
if let Some(cb) = progress {
cb("Fetching orig tarball...", "", 0, 0);
}
fetch_orig_tarball(&info, Some(&package_dir), progress).await?;
}
} else {
println!("No VCS URL found for package {}", package);
return Err(format!("No VCS URL found for package {}", package).into());
}
Ok(())
@@ -166,7 +224,7 @@ mod tests {
let cwd = temp_dir.path();
// Main 'get' command: the one we want to test
get(package, "", series, "", "", Some(cwd)).await.unwrap();
get(package, "", series, "", "", Some(cwd), None).await.unwrap();
let package_dir = cwd.join(package);
assert!(package_dir.exists(), "Package directory not created");

2
src/lib.rs
View File

@@ -1 +1,3 @@
pub mod package_info;
pub type ProgressCallback<'a> = Option<&'a dyn Fn(&str, &str, usize, usize)>;

61
src/main.rs
View File

@@ -1,26 +1,34 @@
use std::env;
use std::error::Error;
use std::collections::HashMap;
extern crate serde;
use serde::Deserialize;
use std::io::Write;
use std::time::Duration;
extern crate clap;
use clap::{arg, command, value_parser, ArgAction, Command};
use clap::{arg, command, Command};
extern crate flate2;
extern crate cmd_lib;
use cmd_lib::{run_cmd};
mod get;
use get::get;
mod changelog;
use changelog::generate_entry;
use log::{info, error};
use indicatif_log_bridge::LogWrapper;
fn main() {
let rt = tokio::runtime::Runtime::new().unwrap();
let logger =
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info"))
.format_timestamp(None)
.format(|buf, record| {
writeln!(buf, "{}", record.args())
})
.build();
let multi = indicatif::MultiProgress::new();
LogWrapper::new(multi.clone(), logger)
.try_init()
.unwrap();
let matches = command!()
.subcommand_required(true)
.disable_version_flag(true)
@@ -59,17 +67,46 @@ fn main() {
let ppa = sub_matches.get_one::<String>("ppa").map(|s| s.as_str()).unwrap_or("");
// Since get is async, we need to block on it
if let Err(e) = rt.block_on(get(package, version, series, "", ppa, None)) {
eprintln!("Error: {}", e);
let pb = multi.add(indicatif::ProgressBar::new(0));
pb.enable_steady_tick(Duration::from_millis(50));
let mut progress_callback = |prefix: &str, msg: &str, progress: usize, total: usize| {
if progress != 0 && total != 0 {
pb.set_style(indicatif::ProgressStyle::default_bar()
.template("> {spinner:.blue} {prefix}\n {msg} [{bar:40.cyan/blue}] {pos}/{len} ({eta})")
.unwrap()
.progress_chars("=> "));
} else {
pb.set_style(indicatif::ProgressStyle::default_bar()
.template("> {spinner:.blue} {prefix}")
.unwrap());
}
if ! prefix.is_empty() {
pb.set_prefix(prefix.to_string());
}
pb.set_message(msg.to_string());
pb.set_length(total as u64);
pb.set_position(progress as u64);
};
if let Err(e) = rt.block_on(get(package, version, series, "", ppa, None, Some(&mut progress_callback))) {
pb.finish_and_clear();
error!("{}", e);
std::process::exit(1);
}
pb.finish_and_clear();
multi.remove(&pb);
info!("Done.");
},
Some(("chlog", sub_matches)) => {
let cwd = std::env::current_dir().unwrap();
let version = sub_matches.get_one::<String>("version").map(|s| s.as_str());
if let Err(e) = generate_entry("debian/changelog", Some(&cwd), version) {
eprintln!("Error: {}", e);
error!("{}", e);
std::process::exit(1);
}

12
src/package_info.rs
View File

@@ -1,10 +1,11 @@
use flate2::read::GzDecoder;
use std::io::Read;
use std::collections::HashMap;
use serde::Deserialize;
use std::error::Error;
use std::path::Path;
use log::debug;
const BASE_URL_UBUNTU: &str = "http://archive.ubuntu.com/ubuntu";
const BASE_URL_DEBIAN: &str = "http://deb.debian.org/debian";
@@ -185,7 +186,7 @@ pub async fn get(package_name: &str, series: &str, pocket: &str) -> Result<Optio
let mut preferred_vcs = None;
if dist == "ubuntu" {
if let Some(lp_url) = check_launchpad_repo(package_name).await? {
println!("Found Launchpad URL: {}", lp_url);
debug!("Found Launchpad URL: {}", lp_url);
preferred_vcs = Some(lp_url);
}
}
@@ -195,12 +196,12 @@ pub async fn get(package_name: &str, series: &str, pocket: &str) -> Result<Optio
let component = "main"; // TODO: Make configurable or detect
let url = get_sources_url(base_url, series, pocket, component);
println!("Fetching sources from: {}", url);
debug!("Fetching sources from: {}", url);
let response = reqwest::get(&url).await?;
let compressed_data = response.bytes().await?;
println!("Downloaded Sources.gz for {}/{}", dist, series);
debug!("Downloaded Sources.gz for {}/{}", dist, series);
if let Some(stanza) = parse_sources(&compressed_data, package_name)? {
if let Some(vcs) = &stanza.vcs_git {
@@ -217,8 +218,7 @@ pub async fn get(package_name: &str, series: &str, pocket: &str) -> Result<Optio
archive_url: archive_url,
}));
} else {
// println!("Package '{}' not found in {}/{}", package, dist, series);
Ok(None)
Err(format!("Package '{}' not found in {}/{}", package_name, dist, series).into())
}
}