vhaudiquet/ecr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add progressbar for extraction and initramfs creation

Browse Source
This commit is contained in:
Valentin Haudiquet
2026-06-17 00:11:12 +02:00
parent a81e699619
commit 4475dff141
2 changed files with 100 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/extract.rs
+76 -93
View File
@@ -1,5 +1,6 @@
use crate::veprintln;
use anyhow::{Context, Result};
use indicatif::{ProgressBar, ProgressStyle};
use std::fs::File;
use std::io::{BufReader, Read};
use std::path::Path;
@@ -130,16 +131,18 @@ fn extract_oci_layer(layer_path: &Path, dest: &Path) -> Result<()> {
let reader = BufReader::new(file);
if layer_name.ends_with(".tar.gz") || layer_name.ends_with(".tgz") {
extract_archive_with_whiteouts(
extract_with_progress(
tar::Archive::new(flate2::read::GzDecoder::new(reader)),
dest,
"Extracting OCI layer",
)
} else if layer_name.ends_with(".tar.xz") || layer_name.ends_with(".txz") {
extract_archive_with_whiteouts(tar::Archive::new(xz2::read::XzDecoder::new(reader)), dest)
extract_with_progress(tar::Archive::new(xz2::read::XzDecoder::new(reader)), dest, "Extracting OCI layer")
} else if layer_name.ends_with(".tar.zst") || layer_name.ends_with(".tar.zstd") {
extract_archive_with_whiteouts(
extract_with_progress(
tar::Archive::new(zstd::stream::read::Decoder::new(reader)?),
dest,
"Extracting OCI layer",
)
} else {
// Fall back to magic-byte detection
@@ -148,63 +151,100 @@ fn extract_oci_layer(layer_path: &Path, dest: &Path) -> Result<()> {
let _ = peek.read_exact(&mut magic); // short reads are fine for detection
drop(peek);
match magic {
[0x1f, 0x8b, ..] => extract_archive_with_whiteouts(
[0x1f, 0x8b, ..] => extract_with_progress(
tar::Archive::new(flate2::read::GzDecoder::new(BufReader::new(File::open(
layer_path,
)?))),
dest,
"Extracting OCI layer",
),
[0xfd, b'7', b'z', b'X', b'Z', 0x00] => extract_archive_with_whiteouts(
[0xfd, b'7', b'z', b'X', b'Z', 0x00] => extract_with_progress(
tar::Archive::new(xz2::read::XzDecoder::new(BufReader::new(File::open(
layer_path,
)?))),
dest,
"Extracting OCI layer",
),
[0x28, 0xb5, 0x2f, 0xfd, ..] => extract_archive_with_whiteouts(
[0x28, 0xb5, 0x2f, 0xfd, ..] => extract_with_progress(
tar::Archive::new(zstd::stream::read::Decoder::new(BufReader::new(
File::open(layer_path)?,
))?),
dest,
"Extracting OCI layer",
),
_ => extract_archive_with_whiteouts(
_ => extract_with_progress(
tar::Archive::new(BufReader::new(File::open(layer_path)?)),
dest,
"Extracting OCI layer",
),
}
}
}
/// Apply one OCI layer archive to `dest`, interpreting Docker whiteout markers:
///
/// - `.wh.<name>` — Delete `<name>` from a lower layer that was already
/// extracted into `dest`.
/// - `.wh..wh..opq` — Opaque whiteout: the directory that contains this entry
/// is new in this layer; delete everything already in that
/// directory from lower layers before applying new content.
///
/// All other entries are extracted normally via `Entry::unpack_in`.
fn extract_archive_with_whiteouts<R: std::io::Read>(
mut archive: tar::Archive<R>,
dest: &Path,
) -> Result<()> {
fn extract_gz<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
let gz_decoder = flate2::read::GzDecoder::new(reader);
let archive = tar::Archive::new(gz_decoder);
extract_with_progress(archive, dest, "Extracting gzip archive")?;
Ok(())
}
fn extract_xz<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
let xz_decoder = xz2::read::XzDecoder::new(reader);
let archive = tar::Archive::new(xz_decoder);
extract_with_progress(archive, dest, "Extracting xz archive")?;
Ok(())
}
fn extract_zst<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
let zst_decoder = zstd::Decoder::new(reader)?;
let archive = tar::Archive::new(zst_decoder);
extract_with_progress(archive, dest, "Extracting zstd archive")?;
Ok(())
}
fn extract_tar<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
let archive = tar::Archive::new(reader);
extract_with_progress(archive, dest, "Extracting tar archive")?;
Ok(())
}
/// Extract tar archive with progress bar, handling whiteout files for OCI layers
fn extract_with_progress<R: std::io::Read>(mut archive: tar::Archive<R>, dest: &Path, msg: &str) -> Result<()> {
let pb = ProgressBar::new_spinner();
pb.set_style(ProgressStyle::default_spinner()
.template("{spinner:.green} {msg} ({pos} files)")
.unwrap());
pb.set_message(msg.to_string());
archive.set_preserve_permissions(true);
archive.set_preserve_ownerships(false);
archive.set_unpack_xattrs(false);
for entry in archive.entries().context("Failed to iterate tar entries")? {
let mut entry = entry.context("Failed to read tar entry")?;
// Clone the path before any mutable borrow of entry (needed for unpack_in)
let entries = archive.entries()
.context("Failed to read archive entries")?;
for entry in entries {
let mut entry = entry.context("Failed to read archive entry")?;
// Clone the path before any mutable borrow of entry
let path = entry.path().context("Invalid tar entry path")?.into_owned();
let filename = path
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default();
// Handle whiteout files (OCI layer markers for deletions)
if filename == ".wh..wh..opq" {
// Opaque whiteout: clear all previously-extracted content in the
// parent directory so only this layer's content is visible.
// Opaque whiteout: clear all previously-extracted content in the parent directory
let parent = path.parent().unwrap_or(Path::new(""));
let dest_dir = dest.join(parent);
if dest_dir.symlink_metadata().is_ok() {
@@ -218,24 +258,26 @@ fn extract_archive_with_whiteouts<R: std::io::Read>(
})?;
}
}
// Do not extract the .wh..wh..opq marker itself.
// Do not extract the .wh..wh..opq marker itself
} else if let Some(real_name) = filename.strip_prefix(".wh.") {
// Regular whiteout: delete the named path from lower layers.
// Regular whiteout: delete the named path from lower layers
let parent = path.parent().unwrap_or(Path::new(""));
let target = dest.join(parent).join(real_name);
// symlink_metadata (lstat) does not follow symlinks, so a dangling
// symlink is correctly detected and removed rather than silently skipped.
if target.symlink_metadata().is_ok() {
remove_path(&target)
.with_context(|| format!("Whiteout: failed to remove {}", target.display()))?;
}
// Do not extract the .wh.* marker itself.
// Do not extract the .wh.* marker itself
} else {
entry
.unpack_in(dest)
entry.unpack_in(dest)
.with_context(|| format!("Failed to extract {}", path.display()))?;
}
pb.inc(1);
}
pb.finish_and_clear();
veprintln!("Extracted {} files", pb.position());
Ok(())
}
@@ -251,62 +293,3 @@ fn remove_path(path: &Path) -> std::io::Result<()> {
std::fs::remove_file(path)
}
}
fn extract_gz<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
let gz_decoder = flate2::read::GzDecoder::new(reader);
let mut archive = tar::Archive::new(gz_decoder);
archive.set_preserve_permissions(true);
archive.set_preserve_ownerships(false);
archive.set_unpack_xattrs(false);
archive
.unpack(dest)
.context("Failed to extract gzip archive")?;
Ok(())
}
fn extract_xz<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
let xz_decoder = xz2::read::XzDecoder::new(reader);
let mut archive = tar::Archive::new(xz_decoder);
archive.set_preserve_permissions(true);
archive.set_preserve_ownerships(false);
archive.set_unpack_xattrs(false);
archive
.unpack(dest)
.context("Failed to extract xz archive")?;
Ok(())
}
fn extract_zst<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
let zst_decoder = zstd::Decoder::new(reader)?;
let mut archive = tar::Archive::new(zst_decoder);
archive.set_preserve_permissions(true);
archive.set_preserve_ownerships(false);
archive.set_unpack_xattrs(false);
archive
.unpack(dest)
.context("Failed to extract zstd archive")?;
Ok(())
}
fn extract_tar<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
let mut archive = tar::Archive::new(reader);
archive.set_preserve_permissions(true);
archive.set_preserve_ownerships(false);
archive.set_unpack_xattrs(false);
archive
.unpack(dest)
.context("Failed to extract tar archive")?;
Ok(())
}
src/qemu_vm.rs
+24 -9
View File
@@ -1,6 +1,7 @@
use crate::veprintln;
use anyhow::{anyhow, Context, Result};
use cpio::{newc, NewcBuilder};
use indicatif::{ProgressBar, ProgressStyle};
use std::io::Write;
use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf};
@@ -147,13 +148,20 @@ fn get_arch_package_suffix(arch: &str) -> &str {
/// Create a gzipped cpio initramfs from a directory
fn create_initramfs(rootfs: &PathBuf) -> Result<PathBuf> {
veprintln!("Creating initramfs from rootfs...");
// Create a temporary file for the initramfs
let initramfs_path = rootfs.parent().unwrap().join("initramfs.cpio.gz");
// Create the cpio archive
let cpio_data = create_cpio_archive(rootfs)?;
// Create progress bar
let pb = ProgressBar::new_spinner();
pb.set_style(ProgressStyle::default_spinner()
.template("{spinner:.green} {msg} ({pos} files)")
.unwrap());
pb.set_message("Scanning rootfs...");
// Create the cpio archive with progress
let cpio_data = create_cpio_archive(rootfs, &pb)?;
pb.set_message("Compressing initramfs...");
// Compress with gzip
let mut output_file = std::fs::File::create(&initramfs_path)
@@ -165,21 +173,25 @@ fn create_initramfs(rootfs: &PathBuf) -> Result<PathBuf> {
encoder.finish()
.context("Failed to finalize gzip compression")?;
veprintln!("Initramfs created: {} bytes (uncompressed)", cpio_data.len());
// Clear progress bar and print message only in verbose mode
pb.finish_and_clear();
veprintln!("Initramfs created: {} bytes, {} files", cpio_data.len(), pb.position());
Ok(initramfs_path)
}
/// Create a newc-format cpio archive from a directory using the cpio crate
fn create_cpio_archive(rootfs: &Path) -> Result<Vec<u8>> {
fn create_cpio_archive(rootfs: &Path, pb: &ProgressBar) -> Result<Vec<u8>> {
let mut archive = Vec::new();
// Collect all entries with their data
let entries = collect_entries(rootfs, rootfs)?;
let entries = collect_entries(rootfs, rootfs, pb)?;
// Collect entry names for checking existence later
let entry_names: Vec<&str> = entries.iter().map(|(n, _, _, _, _)| n.as_str()).collect();
pb.set_message("Writing initramfs...");
// Write each entry using the cpio crate
for (name, mode, mtime, nlink, data) in &entries {
let file_size = data.len() as u32;
@@ -243,7 +255,7 @@ fn create_cpio_archive(rootfs: &Path) -> Result<Vec<u8>> {
}
/// Collect all filesystem entries recursively
fn collect_entries(base: &Path, current: &Path) -> Result<Vec<(String, u32, u32, u32, Vec<u8>)>> {
fn collect_entries(base: &Path, current: &Path, pb: &ProgressBar) -> Result<Vec<(String, u32, u32, u32, Vec<u8>)>> {
let mut entries = Vec::new();
// Read directory entries
@@ -267,6 +279,9 @@ fn collect_entries(base: &Path, current: &Path) -> Result<Vec<(String, u32, u32,
}
};
// Increment progress counter
pb.inc(1);
let file_type = metadata.file_type();
// Determine mode (file type + permissions from filesystem)
@@ -314,7 +329,7 @@ fn collect_entries(base: &Path, current: &Path) -> Result<Vec<(String, u32, u32,
// Recurse into directories
if file_type.is_dir() {
let mut sub_entries = collect_entries(base, &path)?;
let mut sub_entries = collect_entries(base, &path, pb)?;
entries.append(&mut sub_entries);
}
}