feat: add progressbar for extraction and initramfs creation
This commit is contained in:
+76
-93
@@ -1,5 +1,6 @@
|
|||||||
use crate::veprintln;
|
use crate::veprintln;
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
|
use indicatif::{ProgressBar, ProgressStyle};
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::{BufReader, Read};
|
use std::io::{BufReader, Read};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
@@ -130,16 +131,18 @@ fn extract_oci_layer(layer_path: &Path, dest: &Path) -> Result<()> {
|
|||||||
let reader = BufReader::new(file);
|
let reader = BufReader::new(file);
|
||||||
|
|
||||||
if layer_name.ends_with(".tar.gz") || layer_name.ends_with(".tgz") {
|
if layer_name.ends_with(".tar.gz") || layer_name.ends_with(".tgz") {
|
||||||
extract_archive_with_whiteouts(
|
extract_with_progress(
|
||||||
tar::Archive::new(flate2::read::GzDecoder::new(reader)),
|
tar::Archive::new(flate2::read::GzDecoder::new(reader)),
|
||||||
dest,
|
dest,
|
||||||
|
"Extracting OCI layer",
|
||||||
)
|
)
|
||||||
} else if layer_name.ends_with(".tar.xz") || layer_name.ends_with(".txz") {
|
} else if layer_name.ends_with(".tar.xz") || layer_name.ends_with(".txz") {
|
||||||
extract_archive_with_whiteouts(tar::Archive::new(xz2::read::XzDecoder::new(reader)), dest)
|
extract_with_progress(tar::Archive::new(xz2::read::XzDecoder::new(reader)), dest, "Extracting OCI layer")
|
||||||
} else if layer_name.ends_with(".tar.zst") || layer_name.ends_with(".tar.zstd") {
|
} else if layer_name.ends_with(".tar.zst") || layer_name.ends_with(".tar.zstd") {
|
||||||
extract_archive_with_whiteouts(
|
extract_with_progress(
|
||||||
tar::Archive::new(zstd::stream::read::Decoder::new(reader)?),
|
tar::Archive::new(zstd::stream::read::Decoder::new(reader)?),
|
||||||
dest,
|
dest,
|
||||||
|
"Extracting OCI layer",
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
// Fall back to magic-byte detection
|
// Fall back to magic-byte detection
|
||||||
@@ -148,63 +151,100 @@ fn extract_oci_layer(layer_path: &Path, dest: &Path) -> Result<()> {
|
|||||||
let _ = peek.read_exact(&mut magic); // short reads are fine for detection
|
let _ = peek.read_exact(&mut magic); // short reads are fine for detection
|
||||||
drop(peek);
|
drop(peek);
|
||||||
match magic {
|
match magic {
|
||||||
[0x1f, 0x8b, ..] => extract_archive_with_whiteouts(
|
[0x1f, 0x8b, ..] => extract_with_progress(
|
||||||
tar::Archive::new(flate2::read::GzDecoder::new(BufReader::new(File::open(
|
tar::Archive::new(flate2::read::GzDecoder::new(BufReader::new(File::open(
|
||||||
layer_path,
|
layer_path,
|
||||||
)?))),
|
)?))),
|
||||||
dest,
|
dest,
|
||||||
|
"Extracting OCI layer",
|
||||||
),
|
),
|
||||||
[0xfd, b'7', b'z', b'X', b'Z', 0x00] => extract_archive_with_whiteouts(
|
[0xfd, b'7', b'z', b'X', b'Z', 0x00] => extract_with_progress(
|
||||||
tar::Archive::new(xz2::read::XzDecoder::new(BufReader::new(File::open(
|
tar::Archive::new(xz2::read::XzDecoder::new(BufReader::new(File::open(
|
||||||
layer_path,
|
layer_path,
|
||||||
)?))),
|
)?))),
|
||||||
dest,
|
dest,
|
||||||
|
"Extracting OCI layer",
|
||||||
),
|
),
|
||||||
[0x28, 0xb5, 0x2f, 0xfd, ..] => extract_archive_with_whiteouts(
|
[0x28, 0xb5, 0x2f, 0xfd, ..] => extract_with_progress(
|
||||||
tar::Archive::new(zstd::stream::read::Decoder::new(BufReader::new(
|
tar::Archive::new(zstd::stream::read::Decoder::new(BufReader::new(
|
||||||
File::open(layer_path)?,
|
File::open(layer_path)?,
|
||||||
))?),
|
))?),
|
||||||
dest,
|
dest,
|
||||||
|
"Extracting OCI layer",
|
||||||
),
|
),
|
||||||
_ => extract_archive_with_whiteouts(
|
_ => extract_with_progress(
|
||||||
tar::Archive::new(BufReader::new(File::open(layer_path)?)),
|
tar::Archive::new(BufReader::new(File::open(layer_path)?)),
|
||||||
dest,
|
dest,
|
||||||
|
"Extracting OCI layer",
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Apply one OCI layer archive to `dest`, interpreting Docker whiteout markers:
|
fn extract_gz<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
|
||||||
///
|
let gz_decoder = flate2::read::GzDecoder::new(reader);
|
||||||
/// - `.wh.<name>` — Delete `<name>` from a lower layer that was already
|
let archive = tar::Archive::new(gz_decoder);
|
||||||
/// extracted into `dest`.
|
|
||||||
/// - `.wh..wh..opq` — Opaque whiteout: the directory that contains this entry
|
extract_with_progress(archive, dest, "Extracting gzip archive")?;
|
||||||
/// is new in this layer; delete everything already in that
|
|
||||||
/// directory from lower layers before applying new content.
|
Ok(())
|
||||||
///
|
}
|
||||||
/// All other entries are extracted normally via `Entry::unpack_in`.
|
|
||||||
fn extract_archive_with_whiteouts<R: std::io::Read>(
|
fn extract_xz<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
|
||||||
mut archive: tar::Archive<R>,
|
let xz_decoder = xz2::read::XzDecoder::new(reader);
|
||||||
dest: &Path,
|
let archive = tar::Archive::new(xz_decoder);
|
||||||
) -> Result<()> {
|
|
||||||
|
extract_with_progress(archive, dest, "Extracting xz archive")?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn extract_zst<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
|
||||||
|
let zst_decoder = zstd::Decoder::new(reader)?;
|
||||||
|
let archive = tar::Archive::new(zst_decoder);
|
||||||
|
|
||||||
|
extract_with_progress(archive, dest, "Extracting zstd archive")?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn extract_tar<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
|
||||||
|
let archive = tar::Archive::new(reader);
|
||||||
|
|
||||||
|
extract_with_progress(archive, dest, "Extracting tar archive")?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract tar archive with progress bar, handling whiteout files for OCI layers
|
||||||
|
fn extract_with_progress<R: std::io::Read>(mut archive: tar::Archive<R>, dest: &Path, msg: &str) -> Result<()> {
|
||||||
|
let pb = ProgressBar::new_spinner();
|
||||||
|
pb.set_style(ProgressStyle::default_spinner()
|
||||||
|
.template("{spinner:.green} {msg} ({pos} files)")
|
||||||
|
.unwrap());
|
||||||
|
pb.set_message(msg.to_string());
|
||||||
|
|
||||||
archive.set_preserve_permissions(true);
|
archive.set_preserve_permissions(true);
|
||||||
archive.set_preserve_ownerships(false);
|
archive.set_preserve_ownerships(false);
|
||||||
archive.set_unpack_xattrs(false);
|
archive.set_unpack_xattrs(false);
|
||||||
|
|
||||||
for entry in archive.entries().context("Failed to iterate tar entries")? {
|
let entries = archive.entries()
|
||||||
let mut entry = entry.context("Failed to read tar entry")?;
|
.context("Failed to read archive entries")?;
|
||||||
|
|
||||||
// Clone the path before any mutable borrow of entry (needed for unpack_in)
|
for entry in entries {
|
||||||
|
let mut entry = entry.context("Failed to read archive entry")?;
|
||||||
|
|
||||||
|
// Clone the path before any mutable borrow of entry
|
||||||
let path = entry.path().context("Invalid tar entry path")?.into_owned();
|
let path = entry.path().context("Invalid tar entry path")?.into_owned();
|
||||||
|
|
||||||
let filename = path
|
let filename = path
|
||||||
.file_name()
|
.file_name()
|
||||||
.map(|n| n.to_string_lossy().into_owned())
|
.map(|n| n.to_string_lossy().into_owned())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
// Handle whiteout files (OCI layer markers for deletions)
|
||||||
if filename == ".wh..wh..opq" {
|
if filename == ".wh..wh..opq" {
|
||||||
// Opaque whiteout: clear all previously-extracted content in the
|
// Opaque whiteout: clear all previously-extracted content in the parent directory
|
||||||
// parent directory so only this layer's content is visible.
|
|
||||||
let parent = path.parent().unwrap_or(Path::new(""));
|
let parent = path.parent().unwrap_or(Path::new(""));
|
||||||
let dest_dir = dest.join(parent);
|
let dest_dir = dest.join(parent);
|
||||||
if dest_dir.symlink_metadata().is_ok() {
|
if dest_dir.symlink_metadata().is_ok() {
|
||||||
@@ -218,24 +258,26 @@ fn extract_archive_with_whiteouts<R: std::io::Read>(
|
|||||||
})?;
|
})?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Do not extract the .wh..wh..opq marker itself.
|
// Do not extract the .wh..wh..opq marker itself
|
||||||
} else if let Some(real_name) = filename.strip_prefix(".wh.") {
|
} else if let Some(real_name) = filename.strip_prefix(".wh.") {
|
||||||
// Regular whiteout: delete the named path from lower layers.
|
// Regular whiteout: delete the named path from lower layers
|
||||||
let parent = path.parent().unwrap_or(Path::new(""));
|
let parent = path.parent().unwrap_or(Path::new(""));
|
||||||
let target = dest.join(parent).join(real_name);
|
let target = dest.join(parent).join(real_name);
|
||||||
// symlink_metadata (lstat) does not follow symlinks, so a dangling
|
|
||||||
// symlink is correctly detected and removed rather than silently skipped.
|
|
||||||
if target.symlink_metadata().is_ok() {
|
if target.symlink_metadata().is_ok() {
|
||||||
remove_path(&target)
|
remove_path(&target)
|
||||||
.with_context(|| format!("Whiteout: failed to remove {}", target.display()))?;
|
.with_context(|| format!("Whiteout: failed to remove {}", target.display()))?;
|
||||||
}
|
}
|
||||||
// Do not extract the .wh.* marker itself.
|
// Do not extract the .wh.* marker itself
|
||||||
} else {
|
} else {
|
||||||
entry
|
entry.unpack_in(dest)
|
||||||
.unpack_in(dest)
|
|
||||||
.with_context(|| format!("Failed to extract {}", path.display()))?;
|
.with_context(|| format!("Failed to extract {}", path.display()))?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pb.inc(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pb.finish_and_clear();
|
||||||
|
veprintln!("Extracted {} files", pb.position());
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,62 +293,3 @@ fn remove_path(path: &Path) -> std::io::Result<()> {
|
|||||||
std::fs::remove_file(path)
|
std::fs::remove_file(path)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn extract_gz<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
|
|
||||||
let gz_decoder = flate2::read::GzDecoder::new(reader);
|
|
||||||
let mut archive = tar::Archive::new(gz_decoder);
|
|
||||||
|
|
||||||
archive.set_preserve_permissions(true);
|
|
||||||
archive.set_preserve_ownerships(false);
|
|
||||||
archive.set_unpack_xattrs(false);
|
|
||||||
|
|
||||||
archive
|
|
||||||
.unpack(dest)
|
|
||||||
.context("Failed to extract gzip archive")?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn extract_xz<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
|
|
||||||
let xz_decoder = xz2::read::XzDecoder::new(reader);
|
|
||||||
let mut archive = tar::Archive::new(xz_decoder);
|
|
||||||
|
|
||||||
archive.set_preserve_permissions(true);
|
|
||||||
archive.set_preserve_ownerships(false);
|
|
||||||
archive.set_unpack_xattrs(false);
|
|
||||||
|
|
||||||
archive
|
|
||||||
.unpack(dest)
|
|
||||||
.context("Failed to extract xz archive")?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn extract_zst<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
|
|
||||||
let zst_decoder = zstd::Decoder::new(reader)?;
|
|
||||||
let mut archive = tar::Archive::new(zst_decoder);
|
|
||||||
|
|
||||||
archive.set_preserve_permissions(true);
|
|
||||||
archive.set_preserve_ownerships(false);
|
|
||||||
archive.set_unpack_xattrs(false);
|
|
||||||
|
|
||||||
archive
|
|
||||||
.unpack(dest)
|
|
||||||
.context("Failed to extract zstd archive")?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn extract_tar<R: std::io::Read>(reader: R, dest: &Path) -> Result<()> {
|
|
||||||
let mut archive = tar::Archive::new(reader);
|
|
||||||
|
|
||||||
archive.set_preserve_permissions(true);
|
|
||||||
archive.set_preserve_ownerships(false);
|
|
||||||
archive.set_unpack_xattrs(false);
|
|
||||||
|
|
||||||
archive
|
|
||||||
.unpack(dest)
|
|
||||||
.context("Failed to extract tar archive")?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|||||||
+24
-9
@@ -1,6 +1,7 @@
|
|||||||
use crate::veprintln;
|
use crate::veprintln;
|
||||||
use anyhow::{anyhow, Context, Result};
|
use anyhow::{anyhow, Context, Result};
|
||||||
use cpio::{newc, NewcBuilder};
|
use cpio::{newc, NewcBuilder};
|
||||||
|
use indicatif::{ProgressBar, ProgressStyle};
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
use std::os::unix::fs::MetadataExt;
|
use std::os::unix::fs::MetadataExt;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
@@ -147,13 +148,20 @@ fn get_arch_package_suffix(arch: &str) -> &str {
|
|||||||
|
|
||||||
/// Create a gzipped cpio initramfs from a directory
|
/// Create a gzipped cpio initramfs from a directory
|
||||||
fn create_initramfs(rootfs: &PathBuf) -> Result<PathBuf> {
|
fn create_initramfs(rootfs: &PathBuf) -> Result<PathBuf> {
|
||||||
veprintln!("Creating initramfs from rootfs...");
|
|
||||||
|
|
||||||
// Create a temporary file for the initramfs
|
// Create a temporary file for the initramfs
|
||||||
let initramfs_path = rootfs.parent().unwrap().join("initramfs.cpio.gz");
|
let initramfs_path = rootfs.parent().unwrap().join("initramfs.cpio.gz");
|
||||||
|
|
||||||
// Create the cpio archive
|
// Create progress bar
|
||||||
let cpio_data = create_cpio_archive(rootfs)?;
|
let pb = ProgressBar::new_spinner();
|
||||||
|
pb.set_style(ProgressStyle::default_spinner()
|
||||||
|
.template("{spinner:.green} {msg} ({pos} files)")
|
||||||
|
.unwrap());
|
||||||
|
pb.set_message("Scanning rootfs...");
|
||||||
|
|
||||||
|
// Create the cpio archive with progress
|
||||||
|
let cpio_data = create_cpio_archive(rootfs, &pb)?;
|
||||||
|
|
||||||
|
pb.set_message("Compressing initramfs...");
|
||||||
|
|
||||||
// Compress with gzip
|
// Compress with gzip
|
||||||
let mut output_file = std::fs::File::create(&initramfs_path)
|
let mut output_file = std::fs::File::create(&initramfs_path)
|
||||||
@@ -165,21 +173,25 @@ fn create_initramfs(rootfs: &PathBuf) -> Result<PathBuf> {
|
|||||||
encoder.finish()
|
encoder.finish()
|
||||||
.context("Failed to finalize gzip compression")?;
|
.context("Failed to finalize gzip compression")?;
|
||||||
|
|
||||||
veprintln!("Initramfs created: {} bytes (uncompressed)", cpio_data.len());
|
// Clear progress bar and print message only in verbose mode
|
||||||
|
pb.finish_and_clear();
|
||||||
|
veprintln!("Initramfs created: {} bytes, {} files", cpio_data.len(), pb.position());
|
||||||
|
|
||||||
Ok(initramfs_path)
|
Ok(initramfs_path)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Create a newc-format cpio archive from a directory using the cpio crate
|
/// Create a newc-format cpio archive from a directory using the cpio crate
|
||||||
fn create_cpio_archive(rootfs: &Path) -> Result<Vec<u8>> {
|
fn create_cpio_archive(rootfs: &Path, pb: &ProgressBar) -> Result<Vec<u8>> {
|
||||||
let mut archive = Vec::new();
|
let mut archive = Vec::new();
|
||||||
|
|
||||||
// Collect all entries with their data
|
// Collect all entries with their data
|
||||||
let entries = collect_entries(rootfs, rootfs)?;
|
let entries = collect_entries(rootfs, rootfs, pb)?;
|
||||||
|
|
||||||
// Collect entry names for checking existence later
|
// Collect entry names for checking existence later
|
||||||
let entry_names: Vec<&str> = entries.iter().map(|(n, _, _, _, _)| n.as_str()).collect();
|
let entry_names: Vec<&str> = entries.iter().map(|(n, _, _, _, _)| n.as_str()).collect();
|
||||||
|
|
||||||
|
pb.set_message("Writing initramfs...");
|
||||||
|
|
||||||
// Write each entry using the cpio crate
|
// Write each entry using the cpio crate
|
||||||
for (name, mode, mtime, nlink, data) in &entries {
|
for (name, mode, mtime, nlink, data) in &entries {
|
||||||
let file_size = data.len() as u32;
|
let file_size = data.len() as u32;
|
||||||
@@ -243,7 +255,7 @@ fn create_cpio_archive(rootfs: &Path) -> Result<Vec<u8>> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Collect all filesystem entries recursively
|
/// Collect all filesystem entries recursively
|
||||||
fn collect_entries(base: &Path, current: &Path) -> Result<Vec<(String, u32, u32, u32, Vec<u8>)>> {
|
fn collect_entries(base: &Path, current: &Path, pb: &ProgressBar) -> Result<Vec<(String, u32, u32, u32, Vec<u8>)>> {
|
||||||
let mut entries = Vec::new();
|
let mut entries = Vec::new();
|
||||||
|
|
||||||
// Read directory entries
|
// Read directory entries
|
||||||
@@ -267,6 +279,9 @@ fn collect_entries(base: &Path, current: &Path) -> Result<Vec<(String, u32, u32,
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Increment progress counter
|
||||||
|
pb.inc(1);
|
||||||
|
|
||||||
let file_type = metadata.file_type();
|
let file_type = metadata.file_type();
|
||||||
|
|
||||||
// Determine mode (file type + permissions from filesystem)
|
// Determine mode (file type + permissions from filesystem)
|
||||||
@@ -314,7 +329,7 @@ fn collect_entries(base: &Path, current: &Path) -> Result<Vec<(String, u32, u32,
|
|||||||
|
|
||||||
// Recurse into directories
|
// Recurse into directories
|
||||||
if file_type.is_dir() {
|
if file_type.is_dir() {
|
||||||
let mut sub_entries = collect_entries(base, &path)?;
|
let mut sub_entries = collect_entries(base, &path, pb)?;
|
||||||
entries.append(&mut sub_entries);
|
entries.append(&mut sub_entries);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user