deb: record per-phase build timings, shown by pkh deb profile

Time every Phase the build flow announces through enter_phase — the
chroot bootstrap (tarball download, extraction, finalization), the
apt phases, patch application, the build and binary steps and the
artifact retrieval — and land the ordered profile in the record's
manifest of every run, failed builds included: their profile ends
at the phase the error propagated out of.

Timing rides the enter_phase choke point instead of new measurement
sites, so a phase added later profiles itself. The manifest field
is additive (serde default, skipped when absent), so older manifests
keep loading without a schema bump. The coarse timings stay the
source of the deb list columns: they measure the apt transactions
themselves, where the profile measures whole phases.

pkh deb profile renders a record's profile through the same
read-only record discovery as pkh deb log — the newest record of
the tree, or an id prefix as shown by pkh deb list — so
record_for_log becomes the neutral find_record. Phase keys are
storage names, distinct from the display labels; Phase::from_key
turns them back and a test keeps both in sync.
This commit is contained in:
2026-10-02 14:17:15 +02:00
parent 7055d69152
commit 2ab2ff91e9
8 changed files with 552 additions and 38 deletions
+5
View File
@@ -174,6 +174,11 @@ pkh deb list # the records of this tree: id, outcome,
# dep-install and build times, age # dep-install and build times, age
pkh deb log [-f] [<id>] # print a record's build log; -f follows a pkh deb log [-f] [<id>] # print a record's build log; -f follows a
# live build # live build
pkh deb profile [<id>] # a record's per-phase timing profile: how
# long the chroot bootstrap, each apt phase,
# patch application, the build and binary
# steps and the artifact retrieval took, with
# their shares of the total
pkh deb --no-ccache # build without the shared compile cache pkh deb --no-ccache # build without the shared compile cache
``` ```
+29 -7
View File
@@ -1,6 +1,6 @@
use crate::context::{self, Context, ContextConfig}; use crate::context::{self, Context, ContextConfig};
use crate::deb::session::{self, Session}; use crate::deb::session::{self, Session};
use crate::deb::{Phase, enter_phase}; use crate::deb::{BuildProfiler, Phase, enter_phase};
use crate::interrupt::CleanupHookGuard; use crate::interrupt::CleanupHookGuard;
use crate::report::BuildView; use crate::report::BuildView;
use directories::ProjectDirs; use directories::ProjectDirs;
@@ -511,14 +511,20 @@ impl EphemeralContextGuard {
/// * `arch` - Optional target architecture. If provided and different from host, /// * `arch` - Optional target architecture. If provided and different from host,
/// downloads a chroot for that architecture (uses qemu_binfmt transparently) /// downloads a chroot for that architecture (uses qemu_binfmt transparently)
/// * `base_ctx` - The base context to use for creating the chroot /// * `base_ctx` - The base context to use for creating the chroot
/// * `profiler` - The per-phase build profile to feed (the chroot
/// bootstrap's phases of the record's `pkh deb profile`)
pub async fn new_with_context( pub async fn new_with_context(
series: &str, series: &str,
arch: Option<&str>, arch: Option<&str>,
mirror: Option<&str>, mirror: Option<&str>,
base_ctx: Arc<Context>, base_ctx: Arc<Context>,
view: &dyn BuildView, view: &dyn BuildView,
profiler: &mut BuildProfiler,
) -> Result<Self, Box<dyn Error>> { ) -> Result<Self, Box<dyn Error>> {
Self::new_internal(None, series, arch, mirror, base_ctx, view, false, None).await Self::new_internal(
None, series, arch, mirror, base_ctx, view, false, None, profiler,
)
.await
} }
/// Create the ephemeral build context inside `session`'s chroot. /// Create the ephemeral build context inside `session`'s chroot.
@@ -536,6 +542,8 @@ impl EphemeralContextGuard {
/// * `ccache_dir` - The shared ccache directory to mount at the /// * `ccache_dir` - The shared ccache directory to mount at the
/// chroot's `/var/cache/pkh-ccache` (`pkh deb` computes it from the /// chroot's `/var/cache/pkh-ccache` (`pkh deb` computes it from the
/// build identity; `None` leaves the chroot unshared) /// build identity; `None` leaves the chroot unshared)
/// * `profiler` - The per-phase build profile to feed (the chroot
/// bootstrap's phases of the record's `pkh deb profile`)
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
pub async fn new_for_session( pub async fn new_for_session(
session: Session, session: Session,
@@ -546,6 +554,7 @@ impl EphemeralContextGuard {
view: &dyn BuildView, view: &dyn BuildView,
keep_upper: bool, keep_upper: bool,
ccache_dir: Option<&Path>, ccache_dir: Option<&Path>,
profiler: &mut BuildProfiler,
) -> Result<Self, Box<dyn Error>> { ) -> Result<Self, Box<dyn Error>> {
Self::new_internal( Self::new_internal(
Some(session), Some(session),
@@ -556,6 +565,7 @@ impl EphemeralContextGuard {
view, view,
keep_upper, keep_upper,
ccache_dir, ccache_dir,
profiler,
) )
.await .await
} }
@@ -570,6 +580,7 @@ impl EphemeralContextGuard {
view: &dyn BuildView, view: &dyn BuildView,
keep_upper: bool, keep_upper: bool,
ccache_dir: Option<&Path>, ccache_dir: Option<&Path>,
profiler: &mut BuildProfiler,
) -> Result<Self, Box<dyn Error>> { ) -> Result<Self, Box<dyn Error>> {
// Save the globally-installed context so Drop can restore exactly // Save the globally-installed context so Drop can restore exactly
// this handle: concurrent builds install their own ephemeral // this handle: concurrent builds install their own ephemeral
@@ -638,6 +649,7 @@ impl EphemeralContextGuard {
base_ctx.clone(), base_ctx.clone(),
view, view,
ccache_dir, ccache_dir,
profiler,
) )
.await .await
{ {
@@ -711,6 +723,7 @@ impl EphemeralContextGuard {
Arc::clone(&self.ephemeral_ctx) Arc::clone(&self.ephemeral_ctx)
} }
#[allow(clippy::too_many_arguments)]
async fn download_and_extract_chroot( async fn download_and_extract_chroot(
series: &str, series: &str,
arch: Option<&str>, arch: Option<&str>,
@@ -719,6 +732,7 @@ impl EphemeralContextGuard {
ctx: Arc<context::Context>, ctx: Arc<context::Context>,
view: &dyn BuildView, view: &dyn BuildView,
ccache_dir: Option<&Path>, ccache_dir: Option<&Path>,
profiler: &mut BuildProfiler,
) -> Result<Option<fs::File>, Box<dyn Error>> { ) -> Result<Option<fs::File>, Box<dyn Error>> {
// The chroot is per-run scratch: nothing is reused, so never // The chroot is per-run scratch: nothing is reused, so never
// extract over stale content. // extract over stale content.
@@ -778,7 +792,7 @@ impl EphemeralContextGuard {
series, series,
arch arch
); );
enter_phase(view, Phase::PreparingChroot); enter_phase(view, Phase::PreparingChroot, profiler);
Self::download_chroot_tarball(series, arch, mirror, &tarball_path, ctx, view).await?; Self::download_chroot_tarball(series, arch, mirror, &tarball_path, ctx, view).await?;
} else { } else {
log::debug!( log::debug!(
@@ -790,12 +804,19 @@ impl EphemeralContextGuard {
// Extract tarball to chroot directory // Extract tarball to chroot directory
log::debug!("Extracting chroot tarball to {}...", chroot_path.display()); log::debug!("Extracting chroot tarball to {}...", chroot_path.display());
enter_phase(view, Phase::ExtractingChroot); enter_phase(view, Phase::ExtractingChroot, profiler);
Self::extract_tarball(&tarball_path, chroot_path, view)?; Self::extract_tarball(&tarball_path, chroot_path, view)?;
// Create device nodes, bind-mount /proc and share the apt cache // Create device nodes, bind-mount /proc and share the apt cache
let apt_cache_lock = let apt_cache_lock = Self::finalize_chroot(
Self::finalize_chroot(series, arch, chroot_path, ctx_for_devices, view, ccache_dir)?; series,
arch,
chroot_path,
ctx_for_devices,
view,
ccache_dir,
profiler,
)?;
// The bootstrap's tarball digest is recorded by the caller (the // The bootstrap's tarball digest is recorded by the caller (the
// record's fork gate compares it against the cache later). // record's fork gate compares it against the cache later).
@@ -814,13 +835,14 @@ impl EphemeralContextGuard {
ctx: Arc<context::Context>, ctx: Arc<context::Context>,
view: &dyn BuildView, view: &dyn BuildView,
ccache_dir: Option<&Path>, ccache_dir: Option<&Path>,
profiler: &mut BuildProfiler,
) -> Result<Option<fs::File>, Box<dyn Error>> { ) -> Result<Option<fs::File>, Box<dyn Error>> {
if !Self::device_nodes_ready(chroot_path) { if !Self::device_nodes_ready(chroot_path) {
// The tar crate materializes the tarball's device entries as // The tar crate materializes the tarball's device entries as
// empty regular files (see [Self::DEVICE_NODES]), so readiness // empty regular files (see [Self::DEVICE_NODES]), so readiness
// is keyed on the nodes being char devices, not on presence. // is keyed on the nodes being char devices, not on presence.
log::debug!("Creating device nodes in chroot..."); log::debug!("Creating device nodes in chroot...");
enter_phase(view, Phase::FinalizingChroot); enter_phase(view, Phase::FinalizingChroot, profiler);
Self::create_device_nodes(chroot_path, ctx.clone())?; Self::create_device_nodes(chroot_path, ctx.clone())?;
} }
+36 -10
View File
@@ -1,7 +1,7 @@
/// Local binary package building /// Local binary package building
/// Directly calling 'debian/rules' in current context /// Directly calling 'debian/rules' in current context
use crate::context::{Context, ContextCommand, LineSink}; use crate::context::{Context, ContextCommand, LineSink};
use crate::deb::{Phase, enter_phase, find_dsc_file}; use crate::deb::{BuildProfiler, Phase, enter_phase, find_dsc_file};
use crate::logfmt::QuiltClassifier; use crate::logfmt::QuiltClassifier;
use crate::report::BuildView; use crate::report::BuildView;
use log::warn; use log::warn;
@@ -275,6 +275,9 @@ pub async fn build(
// Accumulator of the measured dependency-installation and build // Accumulator of the measured dependency-installation and build
// durations; the caller records it on the build record. // durations; the caller records it on the build record.
timings: &mut session::BuildTimings, timings: &mut session::BuildTimings,
// The per-phase profile the caller records on the build record:
// fed by every enter_phase call of the build flow.
profiler: &mut BuildProfiler,
) -> Result<Vec<PathBuf>, Box<dyn Error>> { ) -> Result<Vec<PathBuf>, Box<dyn Error>> {
let sink: Option<Arc<dyn LineSink>> = view.sink(); let sink: Option<Arc<dyn LineSink>> = view.sink();
@@ -486,7 +489,7 @@ pub async fn build(
// Update package lists // Update package lists
log::debug!("Updating package lists for local build..."); log::debug!("Updating package lists for local build...");
enter_phase(view, Phase::UpdatingPackageLists); enter_phase(view, Phase::UpdatingPackageLists, profiler);
let started = Instant::now(); let started = Instant::now();
let status = cap( let status = cap(
ctx.command("apt-get").envs(env.clone()).arg("update"), ctx.command("apt-get").envs(env.clone()).arg("update"),
@@ -532,7 +535,7 @@ pub async fn build(
cmd.arg(format!("libc6:{arch}")); cmd.arg(format!("libc6:{arch}"));
cmd.arg(format!("libc6-dev:{arch}")); cmd.arg(format!("libc6-dev:{arch}"));
} }
enter_phase(view, Phase::InstallingEssentials); enter_phase(view, Phase::InstallingEssentials, profiler);
let started = Instant::now(); let started = Instant::now();
let status = cap(&mut cmd, &sink).status()?; let status = cap(&mut cmd, &sink).status()?;
timings.add_deps(started.elapsed()); timings.add_deps(started.elapsed());
@@ -571,14 +574,30 @@ pub async fn build(
.unwrap_or(false); .unwrap_or(false);
applied && previous != patch_stamp applied && previous != patch_stamp
}; };
apply_quilt_patches(package_dir_str, &env, ctx.clone(), view, &sink, pop_patches)?; apply_quilt_patches(
package_dir_str,
&env,
ctx.clone(),
view,
&sink,
pop_patches,
profiler,
)?;
if let Some(s) = session { if let Some(s) = session {
s.record_phase("patches", patch_stamp); s.record_phase("patches", patch_stamp);
} }
// Install injected packages if specified // Install injected packages if specified
if !inject_packages.is_empty() { if !inject_packages.is_empty() {
install_injected_packages(inject_packages, &env, ctx.clone(), view, &sink, timings)?; install_injected_packages(
inject_packages,
&env,
ctx.clone(),
view,
&sink,
timings,
profiler,
)?;
} }
// Resolve and install the Build-* dependencies with dpkg's cross // Resolve and install the Build-* dependencies with dpkg's cross
@@ -598,6 +617,7 @@ pub async fn build(
view, view,
&sink, &sink,
timings, timings,
profiler,
)?; )?;
// (The build-dependency phase reruns in every fresh chroot: nothing // (The build-dependency phase reruns in every fresh chroot: nothing
// to journal.) // to journal.)
@@ -669,7 +689,7 @@ pub async fn build(
// Run the build step // Run the build step
log::debug!("Building (debian/rules build) package..."); log::debug!("Building (debian/rules build) package...");
enter_phase(view, Phase::Building); enter_phase(view, Phase::Building, profiler);
let started = Instant::now(); let started = Instant::now();
let status = cap( let status = cap(
rules_command(&ctx, rule_ids.as_ref(), "debian/rules") rules_command(&ctx, rule_ids.as_ref(), "debian/rules")
@@ -699,7 +719,7 @@ pub async fn build(
} }
// Run the 'binary' step to produce deb // Run the 'binary' step to produce deb
enter_phase(view, Phase::ProducingBinaries); enter_phase(view, Phase::ProducingBinaries, profiler);
let started = Instant::now(); let started = Instant::now();
let status = cap( let status = cap(
rules_command(&ctx, rule_ids.as_ref(), "fakeroot") rules_command(&ctx, rule_ids.as_ref(), "fakeroot")
@@ -1005,8 +1025,9 @@ fn install_build_dependencies(
view: &dyn BuildView, view: &dyn BuildView,
sink: &Option<Arc<dyn LineSink>>, sink: &Option<Arc<dyn LineSink>>,
timings: &mut session::BuildTimings, timings: &mut session::BuildTimings,
profiler: &mut BuildProfiler,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
enter_phase(view, Phase::InstallingBuildDeps); enter_phase(view, Phase::InstallingBuildDeps, profiler);
let host_arch = arch.to_string(); let host_arch = arch.to_string();
let build_arch = context_build_arch(&ctx); let build_arch = context_build_arch(&ctx);
@@ -1309,6 +1330,7 @@ fn apply_quilt_patches(
view: &dyn BuildView, view: &dyn BuildView,
sink: &Option<Arc<dyn LineSink>>, sink: &Option<Arc<dyn LineSink>>,
pop_first: bool, pop_first: bool,
profiler: &mut BuildProfiler,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
let series_path = Path::new(package_dir).join("debian/patches/series"); let series_path = Path::new(package_dir).join("debian/patches/series");
if !ctx.exists(&series_path)? { if !ctx.exists(&series_path)? {
@@ -1409,7 +1431,9 @@ fn apply_quilt_patches(
} }
} }
// Apply all patches listed in the series // Apply all patches listed in the series (a custom classifier: the
// phase reports per-patch progress, not the default's raw output)
profiler.enter(Phase::ApplyingPatches);
view.phase( view.phase(
Phase::ApplyingPatches.label(), Phase::ApplyingPatches.label(),
Box::new(QuiltClassifier::new(total_patches)), Box::new(QuiltClassifier::new(total_patches)),
@@ -1488,10 +1512,11 @@ pub(crate) fn install_injected_packages(
view: &dyn BuildView, view: &dyn BuildView,
sink: &Option<Arc<dyn LineSink>>, sink: &Option<Arc<dyn LineSink>>,
timings: &mut session::BuildTimings, timings: &mut session::BuildTimings,
profiler: &mut BuildProfiler,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
log::info!("Installing injected packages: {:?}", packages); log::info!("Installing injected packages: {:?}", packages);
enter_phase(view, Phase::InjectingPackages); enter_phase(view, Phase::InjectingPackages, profiler);
// Separate .deb files from package names // Separate .deb files from package names
let mut deb_files: Vec<String> = Vec::new(); let mut deb_files: Vec<String> = Vec::new();
@@ -1955,6 +1980,7 @@ mod tests {
&crate::report::Quiet, &crate::report::Quiet,
&None, &None,
false, false,
&mut BuildProfiler::new(),
) )
.unwrap(); .unwrap();
} }
+183 -9
View File
@@ -68,6 +68,46 @@ impl Phase {
Phase::RetrievingArtifacts => "Retrieving artifacts", Phase::RetrievingArtifacts => "Retrieving artifacts",
} }
} }
/// Stable machine name of the phase, as recorded in a build
/// record's profile (`pkh deb profile`); unlike [`Self::label`]
/// this is a storage key, never reworded.
pub fn key(&self) -> &'static str {
match self {
Phase::PreparingChroot => "preparing_chroot",
Phase::ExtractingChroot => "extracting_chroot",
Phase::FinalizingChroot => "finalizing_chroot",
Phase::UpdatingPackageLists => "updating_package_lists",
Phase::InstallingEssentials => "installing_essentials",
Phase::ApplyingPatches => "applying_patches",
Phase::InjectingPackages => "injecting_packages",
Phase::InstallingBuildDeps => "installing_build_deps",
Phase::Building => "building",
Phase::ProducingBinaries => "producing_binaries",
Phase::RetrievingArtifacts => "retrieving_artifacts",
}
}
/// The phase `key` names, when it is one of the known machine
/// names; a key written by an older pkh whose phase has since been
/// renamed renders as its raw key.
pub fn from_key(key: &str) -> Option<Phase> {
let phase = match key {
"preparing_chroot" => Phase::PreparingChroot,
"extracting_chroot" => Phase::ExtractingChroot,
"finalizing_chroot" => Phase::FinalizingChroot,
"updating_package_lists" => Phase::UpdatingPackageLists,
"installing_essentials" => Phase::InstallingEssentials,
"applying_patches" => Phase::ApplyingPatches,
"injecting_packages" => Phase::InjectingPackages,
"installing_build_deps" => Phase::InstallingBuildDeps,
"building" => Phase::Building,
"producing_binaries" => Phase::ProducingBinaries,
"retrieving_artifacts" => Phase::RetrievingArtifacts,
_ => return None,
};
Some(phase)
}
} }
/// Default line classifier rewriting a phase's subprocess output /// Default line classifier rewriting a phase's subprocess output
@@ -87,8 +127,57 @@ fn default_classifier(phase: Phase) -> Box<dyn Classifier> {
} }
} }
/// Enter `phase` on the view with its default line classifier /// Records the wall-clock duration of every [`Phase`] the build flow
pub(crate) fn enter_phase(view: &dyn BuildView, phase: Phase) { /// announces: one profile entry per executed phase, in execution
/// order. A phase closes when the next one is entered or at
/// [`BuildProfiler::finish`] — so a failed build's profile ends at the
/// phase the error propagated out of, its entry covering the phase
/// until the failure. Conditional phases that never ran (the chroot
/// tarball was cached, the device nodes were ready) record nothing.
pub(crate) struct BuildProfiler {
/// The phase being timed, since when.
open: Option<(Phase, std::time::Instant)>,
/// Closed phases, in execution order.
phases: Vec<session::PhaseTiming>,
}
impl BuildProfiler {
/// A profiler with nothing recorded yet.
pub(crate) fn new() -> Self {
Self {
open: None,
phases: Vec::new(),
}
}
/// Close the phase being timed (if any) and open `phase`.
pub(crate) fn enter(&mut self, phase: Phase) {
self.close_open();
self.open = Some((phase, std::time::Instant::now()));
}
/// Close the open phase and return the recorded profile.
pub(crate) fn finish(&mut self) -> session::BuildProfile {
self.close_open();
session::BuildProfile {
phases: std::mem::take(&mut self.phases),
}
}
fn close_open(&mut self) {
if let Some((phase, started)) = self.open.take() {
self.phases.push(session::PhaseTiming {
phase: phase.key().to_string(),
millis: started.elapsed().as_millis() as u64,
});
}
}
}
/// Enter `phase` on the view with its default line classifier, timing
/// it into `profiler`
pub(crate) fn enter_phase(view: &dyn BuildView, phase: Phase, profiler: &mut BuildProfiler) {
profiler.enter(phase);
view.phase(phase.label(), default_classifier(phase)); view.phase(phase.label(), default_classifier(phase));
} }
@@ -451,6 +540,10 @@ async fn build_binary_package_impl(
// The shared ccache of this target — skipped with --no-ccache. // The shared ccache of this target — skipped with --no-ccache.
let ccache_dir = (session_enabled && ccache).then(|| session::ccache_dir(&identity)); let ccache_dir = (session_enabled && ccache).then(|| session::ccache_dir(&identity));
let mirror_arg = mirror.as_deref(); let mirror_arg = mirror.as_deref();
// The per-phase wall-clock profile of this run: fed by every
// enter_phase call from the chroot bootstrap to the artifact
// retrieval, recorded on the record at the end of the run.
let mut profiler = BuildProfiler::new();
let mut guard = if let Some(s) = session.clone() { let mut guard = if let Some(s) = session.clone() {
Some( Some(
ephemeral::EphemeralContextGuard::new_for_session( ephemeral::EphemeralContextGuard::new_for_session(
@@ -462,6 +555,7 @@ async fn build_binary_package_impl(
view, view,
keep_upper, keep_upper,
ccache_dir.as_deref(), ccache_dir.as_deref(),
&mut profiler,
) )
.await?, .await?,
) )
@@ -473,6 +567,7 @@ async fn build_binary_package_impl(
mirror_arg, mirror_arg,
base_ctx.clone(), base_ctx.clone(),
view, view,
&mut profiler,
) )
.await?, .await?,
) )
@@ -611,6 +706,7 @@ async fn build_binary_package_impl(
// ephemeral chroot) can carry the on-demand build user. // ephemeral chroot) can carry the on-demand build user.
guard.is_some().then_some(local::BUILD_USER), guard.is_some().then_some(local::BUILD_USER),
&mut timings, &mut timings,
&mut profiler,
) )
.await? .await?
} }
@@ -618,7 +714,7 @@ async fn build_binary_package_impl(
// Retrieve the produced artifacts (binary packages plus the upload // Retrieve the produced artifacts (binary packages plus the upload
// metadata) to the parent directory. // metadata) to the parent directory.
enter_phase(view, Phase::RetrievingArtifacts); enter_phase(view, Phase::RetrievingArtifacts, &mut profiler);
let total_debs = remote_files.len(); let total_debs = remote_files.len();
let mut artifacts = Vec::with_capacity(total_debs); let mut artifacts = Vec::with_capacity(total_debs);
@@ -658,12 +754,17 @@ async fn build_binary_package_impl(
s.set_outcome(session::OUTCOME_FAILED); s.set_outcome(session::OUTCOME_FAILED);
} }
// The measured durations land in the record whatever the outcome: a // The measured durations land in the record whatever the outcome: a
// failed build's partial timings are data too. Nothing measured (the // failed build's partial timings and phase profile are data too.
// run ended before any timed command) records nothing. // Nothing measured (the run ended before any timed command ran)
if let Some(s) = &session // records nothing.
&& !timings.is_empty() if let Some(s) = &session {
{ let profile = profiler.finish();
s.set_timings(timings); if !profile.is_empty() {
s.set_profile(profile);
}
if !timings.is_empty() {
s.set_timings(timings);
}
} }
result result
@@ -1032,6 +1133,59 @@ mod tests {
assert!(!is_archive_source_pinned(mirror, None)); assert!(!is_archive_source_pinned(mirror, None));
} }
/// Every phase's machine key round-trips through `Phase::from_key`:
/// the profile renderer relies on it to show a recorded entry as
/// the phase's human label, and a key/from_key drift would
/// downgrade the row to the raw key.
#[test]
fn phase_keys_round_trip() {
for phase in [
Phase::PreparingChroot,
Phase::ExtractingChroot,
Phase::FinalizingChroot,
Phase::UpdatingPackageLists,
Phase::InstallingEssentials,
Phase::ApplyingPatches,
Phase::InjectingPackages,
Phase::InstallingBuildDeps,
Phase::Building,
Phase::ProducingBinaries,
Phase::RetrievingArtifacts,
] {
assert_eq!(Phase::from_key(phase.key()), Some(phase), "{}", phase.key());
assert_ne!(phase.key(), "", "{}", phase.key());
}
assert_eq!(Phase::from_key("nonesuch"), None);
}
/// The profiler closes each phase when the next opens and the last
/// one at `finish`, recording entries in execution order; an empty
/// run records nothing (the record must not claim an empty
/// profile), and a phase never entered leaves no entry.
#[test]
fn profiler_records_phases_in_execution_order() {
let mut profiler = BuildProfiler::new();
assert!(profiler.finish().is_empty());
let mut profiler = BuildProfiler::new();
profiler.enter(Phase::PreparingChroot);
std::thread::sleep(std::time::Duration::from_millis(5));
profiler.enter(Phase::UpdatingPackageLists);
std::thread::sleep(std::time::Duration::from_millis(2));
profiler.enter(Phase::Building);
std::thread::sleep(std::time::Duration::from_millis(2));
let profile = profiler.finish();
let phases: Vec<&str> = profile.phases.iter().map(|p| p.phase.as_str()).collect();
assert_eq!(
phases,
vec!["preparing_chroot", "updating_package_lists", "building"]
);
assert!(profile.total_millis() >= 9, "{profile:?}");
// finish() closes the last phase: nothing stays open afterwards.
assert!(profiler.finish().is_empty());
}
/// An unshare context mapped over `chroot_root`, parented on a local /// An unshare context mapped over `chroot_root`, parented on a local
/// context like the ephemeral build contexts are: exists/list_files/ /// context like the ephemeral build contexts are: exists/list_files/
/// is_dir answer through the path mapping, no namespace privileges /// is_dir answer through the path mapping, no namespace privileges
@@ -1183,6 +1337,26 @@ mod tests {
.expect("Cannot build binary package (deb)"); .expect("Cannot build binary package (deb)");
log::info!("Successfully built binary package"); log::info!("Successfully built binary package");
// The run's record carries the per-phase timing profile: the
// metadata-only record of a plain run keeps the manifest too.
// (The record's host tree is the package directory the build
// was pointed at.)
let tree = cwd.canonicalize().unwrap_or_else(|_| cwd.to_path_buf());
let record =
crate::deb::session::find_record(&crate::deb::session::sessions_root(), &tree, None)
.expect("the build must leave a record for this tree");
let profile = record
.manifest()
.profile
.expect("the record must carry a phase profile");
assert!(
profile
.phases
.iter()
.any(|p| p.phase == "building" && p.millis > 0),
"{profile:?}"
);
// Check that the .deb files are present // Check that the .deb files are present
let parent_dir = cwd.parent().expect("Cannot find parent directory"); let parent_dir = cwd.parent().expect("Cannot find parent directory");
let deb_files = std::fs::read_dir(parent_dir) let deb_files = std::fs::read_dir(parent_dir)
+266 -9
View File
@@ -171,6 +171,43 @@ impl BuildTimings {
} }
} }
/// One executed phase of a build profile: the stable machine name of a
/// [`crate::deb::Phase`] ([`crate::deb::Phase::key`]) and the
/// wall-clock milliseconds it ran.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PhaseTiming {
/// Stable machine name of the phase.
pub phase: String,
/// Wall-clock milliseconds the phase ran.
pub millis: u64,
}
/// Per-phase wall-clock profile of one build, recorded in the record's
/// manifest and shown by `pkh deb profile`: one entry per executed
/// phase, in execution order. A phase the flow skipped (the chroot
/// tarball was already cached, the device nodes were ready) has no
/// entry; a failed build's profile ends at the phase the error
/// propagated out of.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct BuildProfile {
/// Executed phases with their durations, in execution order.
pub phases: Vec<PhaseTiming>,
}
impl BuildProfile {
/// Whether nothing was recorded: the build ended before any phase
/// produced a measurement, and the record must not claim an empty
/// profile.
pub fn is_empty(&self) -> bool {
self.phases.is_empty()
}
/// Total milliseconds across the recorded phases.
pub fn total_millis(&self) -> u64 {
self.phases.iter().map(|p| p.millis).sum()
}
}
/// The record manifest: identity, lineage, outcome and tree state. /// The record manifest: identity, lineage, outcome and tree state.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SessionManifest { pub struct SessionManifest {
@@ -223,6 +260,11 @@ pub struct SessionManifest {
/// record predates the field). /// record predates the field).
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub timings: Option<BuildTimings>, pub timings: Option<BuildTimings>,
/// Per-phase wall-clock profile of this record's build (`None`: not
/// recorded — the record predates the field, or the build ended
/// before any phase produced a measurement).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub profile: Option<BuildProfile>,
} }
/// State shared through the [`Session`] handle: the manifest plus the /// State shared through the [`Session`] handle: the manifest plus the
@@ -374,6 +416,14 @@ impl Session {
}); });
} }
/// Record the per-phase wall-clock profile of this record's build
/// and persist it.
pub fn set_profile(&self, profile: BuildProfile) {
self.with_manifest(|manifest| {
manifest.profile = Some(profile);
});
}
/// Run `f` over the manifest and persist the result; failures are /// Run `f` over the manifest and persist the result; failures are
/// logged and never abort the build (the journal is a cache). /// logged and never abort the build (the journal is a cache).
fn with_manifest(&self, f: impl FnOnce(&mut SessionManifest)) { fn with_manifest(&self, f: impl FnOnce(&mut SessionManifest)) {
@@ -942,6 +992,7 @@ fn fresh_manifest(
kept_upper: false, kept_upper: false,
build_resume: false, build_resume: false,
timings: None, timings: None,
profile: None,
} }
} }
@@ -1102,6 +1153,7 @@ pub(crate) fn fork_record_in(
kept_upper: false, kept_upper: false,
build_resume: false, build_resume: false,
timings: None, timings: None,
profile: None,
})?; })?;
if carry_upper { if carry_upper {
@@ -1418,6 +1470,115 @@ fn format_duration(secs: u64) -> String {
} }
} }
/// Human-readable duration of a profiled phase: the phases can be
/// shorter than a second (artifact retrieval), so sub-second entries
/// render as milliseconds and the rest through [`format_duration`].
fn format_phase_millis(millis: u64) -> String {
if millis < 1000 {
format!("{millis}ms")
} else {
format_duration(millis / 1000)
}
}
/// Render `pkh deb profile` for `record`: its identity in the header,
/// one row per recorded phase with its duration and its share of the
/// total, and — for a failed build — the phase the error surfaced
/// from. A record without a profile (predating the field, or a build
/// interrupted before any phase completed) says so.
pub fn render_record_profile(record: &Session) -> String {
let manifest = record.manifest();
let cross = if manifest.identity.cross { "*" } else { "" };
let mut out = format!(
"Profile of build {} — {} {}, {}/{}{}, {}",
manifest.id,
manifest.identity.package,
manifest.tree_version,
manifest.identity.series,
manifest.identity.arch,
cross,
manifest.last_outcome
);
if let Some(base) = &manifest.base {
out.push_str(&format!(" (fork of {base})"));
}
out.push_str(":\n");
let Some(profile) = &manifest.profile else {
out.push_str("No phase profile recorded for this build.\n");
return out;
};
if profile.is_empty() {
out.push_str("No phase profile recorded for this build.\n");
return out;
}
let total = profile.total_millis();
let rows: Vec<Vec<String>> = profile
.phases
.iter()
.map(|entry| {
let label = crate::deb::Phase::from_key(&entry.phase)
.map(|phase| phase.label().to_string())
.unwrap_or_else(|| entry.phase.clone());
let share = entry
.millis
.checked_mul(100)
.and_then(|scaled| scaled.checked_div(total.max(1)))
.map(|share| format!("{share}%"))
.unwrap_or_else(|| "-".to_string());
vec![label, format_phase_millis(entry.millis), share]
})
.collect();
let headers = ["PHASE", "TIME", "SHARE"];
let widths: Vec<usize> = headers
.iter()
.enumerate()
.map(|(i, header)| {
rows.iter()
.map(|row| row[i].chars().count())
.max()
.unwrap_or(0)
.max(header.len())
})
.collect();
let line = |cells: &[String]| {
cells
.iter()
.enumerate()
.map(|(i, cell)| {
if i == 0 {
format!("{cell:<width$}", width = widths[i])
} else {
format!("{cell:>width$}", width = widths[i])
}
})
.collect::<Vec<_>>()
.join(" ")
};
out.push_str(&line(
&headers.iter().map(|h| h.to_string()).collect::<Vec<_>>(),
));
out.push('\n');
for row in &rows {
out.push_str(&line(row));
out.push('\n');
}
out.push_str(&line(&[
"Total".to_string(),
format_phase_millis(total),
String::new(),
]));
out.push('\n');
if manifest.last_outcome == OUTCOME_FAILED
&& let Some(last) = profile.phases.last()
&& let Some(phase) = crate::deb::Phase::from_key(&last.phase)
{
out.push_str(&format!("The build failed during \"{}\".\n", phase.label()));
}
out
}
/// Render the `pkh deb list` table. /// Render the `pkh deb list` table.
pub fn render_session_list(tree: &Path, rows: &[SessionListRow]) -> String { pub fn render_session_list(tree: &Path, rows: &[SessionListRow]) -> String {
let mut out = String::new(); let mut out = String::new();
@@ -1490,11 +1651,12 @@ pub fn render_session_list(tree: &Path, rows: &[SessionListRow]) -> String {
// Build log (`pkh deb log`) // Build log (`pkh deb log`)
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
/// Resolve the record whose log `pkh deb log` prints: with no id, the /// Resolve the record a read-only deb query targets (`pkh deb log`,
/// newest record of `tree`; with an id (or unique prefix), that exact /// `pkh deb profile`): with no id, the newest record of `tree`; with
/// record of `tree`. `root` is the sessions root (`sessions_root()` in /// an id (or unique prefix), that exact record of `tree`. `root` is
/// production, a fixture root in tests). /// the sessions root (`sessions_root()` in production, a fixture root
pub fn record_for_log( /// in tests).
pub fn find_record(
root: &Path, root: &Path,
tree: &Path, tree: &Path,
id: Option<&str>, id: Option<&str>,
@@ -2364,6 +2526,18 @@ mod tests {
deps_secs: 41, deps_secs: 41,
build_secs: 9020, build_secs: 9020,
}), }),
profile: Some(BuildProfile {
phases: vec![
PhaseTiming {
phase: "preparing_chroot".to_string(),
millis: 38_200,
},
PhaseTiming {
phase: "building".to_string(),
millis: 125_000,
},
],
}),
}; };
let base = tempfile::tempdir().unwrap(); let base = tempfile::tempdir().unwrap();
save_manifest(base.path(), &manifest).unwrap(); save_manifest(base.path(), &manifest).unwrap();
@@ -2378,11 +2552,13 @@ mod tests {
object.remove("tarball_sha256"); object.remove("tarball_sha256");
object.remove("kept_upper"); object.remove("kept_upper");
object.remove("timings"); object.remove("timings");
object.remove("profile");
let older: SessionManifest = serde_json::from_value(value).unwrap(); let older: SessionManifest = serde_json::from_value(value).unwrap();
assert_eq!(older.base, None); assert_eq!(older.base, None);
assert_eq!(older.tarball_sha256, ""); assert_eq!(older.tarball_sha256, "");
assert!(!older.kept_upper); assert!(!older.kept_upper);
assert_eq!(older.timings, None); assert_eq!(older.timings, None);
assert_eq!(older.profile, None);
// A foreign schema is not discoverable. // A foreign schema is not discoverable.
let mut foreign = manifest.clone(); let mut foreign = manifest.clone();
@@ -2484,6 +2660,87 @@ mod tests {
assert_eq!(format_duration(277_200), "3d05h"); assert_eq!(format_duration(277_200), "3d05h");
} }
/// The per-phase profile round-trips through the manifest, and
/// `pkh deb profile` renders each recorded phase under its human
/// label with its share of the total, the total itself, and — for a
/// failed build — the phase the error surfaced from; a record
/// without a profile says so instead.
#[test]
fn profile_persists_and_renders() {
let (root, _tarball) = sessions_fixture();
let identity = test_identity();
let host_tree = root.path().join("tree");
let record = metadata_record(root.path(), &host_tree, &identity, "1.0", OUTCOME_FAILED);
assert_eq!(record.manifest().profile, None);
// No profile recorded: the renderer explains rather than fails.
assert!(
render_record_profile(&record).contains("No phase profile recorded"),
"{}",
render_record_profile(&record)
);
let profile = BuildProfile {
phases: vec![
// An unknown key (an older pkh's phase since renamed)
// renders as its raw key instead of disappearing.
PhaseTiming {
phase: "retired_phase".to_string(),
millis: 500,
},
PhaseTiming {
phase: "preparing_chroot".to_string(),
millis: 10_000,
},
PhaseTiming {
phase: "updating_package_lists".to_string(),
millis: 5_000,
},
PhaseTiming {
phase: "installing_build_deps".to_string(),
millis: 75_000,
},
PhaseTiming {
phase: "building".to_string(),
millis: 10_000,
},
],
};
record.set_profile(profile.clone());
assert_eq!(record.manifest().profile, Some(profile.clone()));
let reloaded = open_from_dir(record.root().to_path_buf()).unwrap();
assert_eq!(reloaded.manifest().profile, Some(profile));
let rendered = render_record_profile(&record);
assert!(rendered.contains("Profile of build"), "{rendered}");
assert!(rendered.contains("1.0"), "{rendered}");
assert!(rendered.contains("failed"), "{rendered}");
// Labels come from the known keys; the unknown key stays raw.
assert!(rendered.contains("Preparing chroot"), "{rendered}");
assert!(rendered.contains("Updating package lists"), "{rendered}");
assert!(
rendered.contains("Installing build dependencies"),
"{rendered}"
);
assert!(rendered.contains("Building package"), "{rendered}");
assert!(rendered.contains("retired_phase"), "{rendered}");
// Shares of the 100_500 ms total, and the total itself.
assert!(rendered.contains("9%"), "{rendered}");
assert!(rendered.contains("74%"), "{rendered}");
assert!(rendered.contains("500ms"), "{rendered}");
assert!(rendered.contains("1m40s"), "{rendered}");
// A failed build's profile names the phase it died in (the last
// recorded one).
assert!(
rendered.contains("failed during \"Building package\""),
"{rendered}"
);
assert_eq!(format_phase_millis(0), "0ms");
assert_eq!(format_phase_millis(740), "740ms");
assert_eq!(format_phase_millis(1000), "1s");
assert_eq!(format_phase_millis(100_500), "1m40s");
}
// ------------------------------------------------------------------- // -------------------------------------------------------------------
// Record layout, fork gate, retention, sweep and logs // Record layout, fork gate, retention, sweep and logs
// ------------------------------------------------------------------- // -------------------------------------------------------------------
@@ -2915,20 +3172,20 @@ mod tests {
drop(newest); drop(newest);
// No id: the newest record of the tree. // No id: the newest record of the tree.
let record = record_for_log(root.path(), &host_tree, None).unwrap(); let record = find_record(root.path(), &host_tree, None).unwrap();
let mut out = Vec::new(); let mut out = Vec::new();
print_record_log(&record, false, &mut out).unwrap(); print_record_log(&record, false, &mut out).unwrap();
assert_eq!(String::from_utf8(out).unwrap(), "newest build log\n"); assert_eq!(String::from_utf8(out).unwrap(), "newest build log\n");
// The exact id of the older record; the record is terminal, so // The exact id of the older record; the record is terminal, so
// --follow just prints. // --follow just prints.
let record = record_for_log(root.path(), &host_tree, Some(&first_id)).unwrap(); let record = find_record(root.path(), &host_tree, Some(&first_id)).unwrap();
let mut out = Vec::new(); let mut out = Vec::new();
print_record_log(&record, true, &mut out).unwrap(); print_record_log(&record, true, &mut out).unwrap();
assert_eq!(String::from_utf8(out).unwrap(), "first build log\n"); assert_eq!(String::from_utf8(out).unwrap(), "first build log\n");
// A wrong id is an error. // A wrong id is an error.
assert!(record_for_log(root.path(), &host_tree, Some("nonesuch")).is_err()); assert!(find_record(root.path(), &host_tree, Some("nonesuch")).is_err());
} }
/// A record whose build captured no line yet streams an empty log. /// A record whose build captured no line yet streams an empty log.
@@ -2940,7 +3197,7 @@ mod tests {
let record = new_record_in(root.path(), &host_tree, &identity, "1.0").unwrap(); let record = new_record_in(root.path(), &host_tree, &identity, "1.0").unwrap();
record.set_outcome(OUTCOME_SUCCESS); record.set_outcome(OUTCOME_SUCCESS);
drop(record); drop(record);
let record = record_for_log(root.path(), &host_tree, None).unwrap(); let record = find_record(root.path(), &host_tree, None).unwrap();
let mut out = Vec::new(); let mut out = Vec::new();
print_record_log(&record, true, &mut out).unwrap(); print_record_log(&record, true, &mut out).unwrap();
assert_eq!(out.len(), 0); assert_eq!(out.len(), 0);
+28 -2
View File
@@ -392,6 +392,12 @@ fn main() {
.arg(arg!([id] "Record id (default: the newest record of this tree)").required(false)) .arg(arg!([id] "Record id (default: the newest record of this tree)").required(false))
.arg(arg!(-f --follow "Keep reading the log as the build writes it")), .arg(arg!(-f --follow "Keep reading the log as the build writes it")),
) )
.subcommand(
Command::new("profile")
.about("Print a recorded build's per-phase timing profile")
.long_about("Print the timing profile of a recorded build: how long every phase took, from the chroot bootstrap (tarball download, extraction, finalization) through the apt phases (package lists, essential packages, injected packages, build dependencies) and patch application to the build and binary steps and the artifact retrieval, each with its share of the total.\nWithout an ID, the newest record of this tree; with an ID (as shown by 'pkh deb list'), that exact record. A failed build's profile ends at the phase the error surfaced from.")
.arg(arg!([id] "Record id (default: the newest record of this tree)").required(false)),
)
) )
.subcommand( .subcommand(
Command::new("test") Command::new("test")
@@ -910,10 +916,10 @@ fn main() {
// `pkh deb log`: print a recorded build's log (optionally // `pkh deb log`: print a recorded build's log (optionally
// following a live build); read-only too. // following a live build); read-only too.
if let Some(("log", log_matches)) = sub_matches.subcommand() { if let Some(("log", log_matches)) = sub_matches.subcommand() {
let tree = cwd.canonicalize().unwrap_or(cwd.clone()); let tree = cwd.canonicalize().unwrap_or_else(|_| cwd.clone());
let id = log_matches.get_one::<String>("id").map(String::as_str); let id = log_matches.get_one::<String>("id").map(String::as_str);
let follow = log_matches.get_flag("follow"); let follow = log_matches.get_flag("follow");
let record = match pkh::deb::session::record_for_log( let record = match pkh::deb::session::find_record(
&pkh::deb::session::sessions_root(), &pkh::deb::session::sessions_root(),
&tree, &tree,
id, id,
@@ -935,6 +941,26 @@ fn main() {
return; return;
} }
// `pkh deb profile`: print a recorded build's per-phase
// timing profile; read-only too.
if let Some(("profile", profile_matches)) = sub_matches.subcommand() {
let tree = cwd.canonicalize().unwrap_or_else(|_| cwd.clone());
let id = profile_matches.get_one::<String>("id").map(String::as_str);
let record = match pkh::deb::session::find_record(
&pkh::deb::session::sessions_root(),
&tree,
id,
) {
Ok(record) => record,
Err(e) => {
error!("{e}");
std::process::exit(1);
}
};
print!("{}", pkh::deb::session::render_record_profile(&record));
return;
}
// Ctrl+C during the build must say what happened and release the // Ctrl+C during the build must say what happened and release the
// ephemeral chroot instead of dying on the default disposition. // ephemeral chroot instead of dying on the default disposition.
// The live view (when enabled) registers its own reporter on top // The live view (when enabled) registers its own reporter on top
+2
View File
@@ -11,6 +11,7 @@ use std::error::Error;
use std::sync::Arc; use std::sync::Arc;
use crate::context::Context; use crate::context::Context;
use crate::deb::BuildProfiler;
use crate::deb::ephemeral::EphemeralContextGuard; use crate::deb::ephemeral::EphemeralContextGuard;
use crate::report::BuildView; use crate::report::BuildView;
use crate::test::options::TestMode; use crate::test::options::TestMode;
@@ -44,6 +45,7 @@ impl Bed {
None, None,
base_ctx, base_ctx,
view, view,
&mut BuildProfiler::new(),
) )
.await?; .await?;
Ok(Bed::Chroot(guard)) Ok(Bed::Chroot(guard))
+3 -1
View File
@@ -561,7 +561,8 @@ async fn configure_testbed(
} }
if !opts.inject.is_empty() { if !opts.inject.is_empty() {
// Testbed setup is not a timed build: the durations go nowhere. // Testbed setup is not a profiled build: the durations and the
// phase profile go nowhere.
crate::deb::local::install_injected_packages( crate::deb::local::install_injected_packages(
&opts.inject, &opts.inject,
env, env,
@@ -569,6 +570,7 @@ async fn configure_testbed(
view, view,
&None, &None,
&mut crate::deb::session::BuildTimings::default(), &mut crate::deb::session::BuildTimings::default(),
&mut crate::deb::BuildProfiler::new(),
)?; )?;
} }
Ok(()) Ok(())