feat(krun): run the library on macOS

The engine is libkrun.1.dylib, dlopen'd from the Homebrew kegs when
the loader's own search fails (bare sonames never reach the brew
prefix); the probe becomes a per-platform hypervisor probe — KVM on
Linux, sysctl + a real hv_vm_create on macOS, so an entitlement
denial (HV_DENIED) names the re-sign fix instead of surfacing as an
opaque krun_start_enter EINVAL. Tool lookup (mke2fs, debugfs) gains
the same keg fallback, and dyld_fallback_dirs() collects the kegs the
bare-soname dlopens (libkrunfw, libarchive) need in
DYLD_FALLBACK_LIBRARY_PATH. The machine child's parent-death contract
moves to a kqueue NOTE_EXIT watcher: macOS has no PR_SET_PDEATHSIG,
and SIGKILL-on-parent-exit is what keeps a killed caller from
stranding a booted machine.
This commit is contained in:
vhaudiquet committed 2026-10-31 18:09:11 -05:00
1 parent a879631c73
commit e075d52f91
7 files changed
+419 -38

No files matched your search

+21 -6
View File
@@ -116,8 +116,15 @@ fn cmd_probe() -> std::process::ExitCode {
let mut ok = true;
print!("kvm: ");
match krun::kvm_probe() {
print!(
"{}: ",
if cfg!(target_os = "macos") {
"hypervisor"
} else {
"kvm"
}
);
match krun::hypervisor_probe() {
Ok(()) => println!("ok"),
Err(err) => {
ok = false;
@@ -145,12 +152,20 @@ fn cmd_probe() -> std::process::ExitCode {
print!("e2fsprogs: ");
let mut tools_ok = true;
for tool in ["mkfs.ext4", "debugfs"] {
match std::process::Command::new(tool).arg("-V").output() {
Ok(_) => {}
Err(err) => {
match ecrou::utils::find_tool(tool) {
Some(path) => match std::process::Command::new(path).arg("-V").output() {
Ok(_) => {}
Err(err) => {
ok = false;
tools_ok = false;
println!("{tool} not runnable ({err}); the rootfs image cannot be built");
break;
}
},
None => {
ok = false;
tools_ok = false;
println!("missing {tool} ({err}); the rootfs image cannot be built");
println!("missing {tool}; install e2fsprogs, the rootfs image cannot be built");
break;
}
}
+26
View File
@@ -34,6 +34,7 @@ const KVM_NESTED_HINT: &str = "only actionable on x86_64 hosts";
/// Errors returned by the ecrou library API.
#[derive(Debug, thiserror::Error)]
#[allow(clippy::enum_variant_names)] // KvmUnavailable is the platform's word; HvUnavailable is macOS's
pub enum Error {
/// No working KVM on this host, and ecrou has no software-emulation
/// fallback. The remediation names the usual causes in order; the
@@ -49,6 +50,18 @@ pub enum Error {
)]
KvmUnavailable(#[source] std::io::Error),
/// No hardware virtualization on this Mac, and ecrou has no
/// software-emulation fallback: libkrun's macOS backend runs on
/// the Hypervisor.framework, which means Apple Silicon and
/// macOS 14 or newer.
#[error(
"Hypervisor.framework is not available\n\n\
ecrou boots microVMs and needs hardware virtualization:\n\
\x20 host: an Apple Silicon Mac (M1 or later)\n\
\x20 os: macOS 14 or newer"
)]
HvUnavailable(#[source] std::io::Error),
/// The registry publishes no linux manifest for the requested
/// architecture; `available` lists the architectures it does serve.
#[error("No manifest found for architecture '{arch}'. Available: {available}")]
@@ -92,6 +105,19 @@ mod tests {
assert!(printed.contains("Permission denied"), "{printed}");
}
#[cfg(target_os = "macos")]
#[test]
fn hv_unavailable_renders_remediation() {
let msg = Error::HvUnavailable(std::io::Error::other("no hvf")).to_string();
assert!(msg.contains("Hypervisor.framework is not available"));
assert!(msg.contains("Apple Silicon"));
let printed = format!(
"{:#}",
anyhow::Error::new(Error::HvUnavailable(std::io::Error::other("no hvf")))
);
assert!(printed.contains("no hvf"), "{printed}");
}
#[test]
fn a_typed_error_survives_the_anyhow_conversion() {
let raised =
+183 -19
View File
@@ -1,13 +1,14 @@
//! Runtime bindings to libkrun's stable C ABI.
//!
//! The library (`libkrun.so.1`) loads through dlopen so a missing
//! install is a remediation message rather than a link error. The
//! The library (`libkrun.so.1` on Linux, `libkrun.1.dylib` on macOS)
//! loads through dlopen so a missing install is a remediation message
//! rather than a link error. The
//! symbol signatures mirror the `stable-1.19.x` headers; the main
//! branch is mid-rewrite to a vtable API and is not the target.
//!
//! The founding rule holds for this module too: libkrun's VMM runs in
//! our process and touches only files, sockets, and `/dev/kvm`, all
//! inside the confinement budget (SPEC.md).
//! our process and touches only files and sockets (plus `/dev/kvm` on
//! Linux), all inside the confinement budget (SPEC.md).
use std::ffi::{c_char, CString};
use std::os::unix::ffi::OsStrExt;
@@ -19,16 +20,41 @@ use crate::Error;
/// The ABI version libkrun reports for a working `/dev/kvm`
/// (`KVM_GET_API_VERSION`, the first request of the KVMIO range).
#[cfg(target_os = "linux")]
const KVM_GET_API_VERSION: libc::c_ulong = 0xAE00;
#[cfg(target_os = "linux")]
const KVM_API_VERSION_EXPECTED: libc::c_int = 12;
#[cfg(target_os = "macos")]
pub const SONAME: &str = "libkrun.1.dylib";
#[cfg(not(target_os = "macos"))]
pub const SONAME: &str = "libkrun.so.1";
/// Remediation shown when libkrun cannot be loaded: distro packaging is
/// still sparse (Fedora carries it; Debian and Ubuntu do not yet).
/// Remediation shown when libkrun cannot be loaded: on macOS the
/// official Homebrew tap bottles the library; Linux distro packaging
/// is still sparse (Fedora carries it; Debian and Ubuntu do not yet).
#[cfg(target_os = "macos")]
const LOAD_REMEDIATION: &str = "install it with `brew install libkrun/krun/libkrun`, \
or point DYLD_LIBRARY_PATH at the directory holding it";
#[cfg(not(target_os = "macos"))]
const LOAD_REMEDIATION: &str = "install libkrun, or point LD_LIBRARY_PATH \
at the directory holding it (a snap stages it next to the binary)";
/// The paths dlopen is tried against, in order. A bare soname relies
/// on the loader's own search (LD_LIBRARY_PATH on Linux; on macOS it
/// never reaches the Homebrew prefix, so the keg paths are named).
fn load_candidates() -> Vec<std::path::PathBuf> {
let mut candidates = vec![std::path::PathBuf::from(SONAME)];
for prefix in crate::utils::BREW_PREFIXES {
candidates.push(
std::path::Path::new(prefix)
.join("opt/libkrun/lib")
.join(SONAME),
);
}
candidates
}
macro_rules! symbol {
($lib:expr, $name:expr, $ty:ty) => {
symbol!($lib, $name, $ty, "libkrun too old?")
@@ -65,11 +91,24 @@ pub struct Krun {
impl Krun {
/// dlopen libkrun, failing with the install remediation.
pub fn load() -> Result<Self> {
// Safety: dlopen of a fixed SONAME; the handle lives in `self`
// and every symbol below is used through it.
let lib = unsafe { libloading::Library::new(SONAME) }
.with_context(|| format!("cannot load {SONAME}: {LOAD_REMEDIATION}"))?;
Ok(Krun { lib })
let candidates = load_candidates();
let mut last: Option<anyhow::Error> = None;
for candidate in &candidates {
// Safety: dlopen of a fixed name; the handle lives in
// `self` and every symbol below is used through it.
let lib = unsafe { libloading::Library::new(candidate) };
match lib {
Ok(lib) => return Ok(Krun { lib }),
Err(err) => last = Some(anyhow::Error::new(err)),
}
}
let tried = candidates
.iter()
.map(|p| p.display().to_string())
.collect::<Vec<_>>()
.join(", ");
Err(last.unwrap_or_else(|| anyhow::anyhow!("no library name to try")))
.with_context(|| format!("cannot load {SONAME} (tried {tried}): {LOAD_REMEDIATION}"))
}
/// `krun_set_log_level`
@@ -327,9 +366,12 @@ fn path_cstring(path: &Path) -> Result<CString> {
.with_context(|| format!("path contains a NUL: {}", path.display()))
}
/// Probe KVM the crun way: open `/dev/kvm` read-write, then check
/// `KVM_GET_API_VERSION` returns 12.
pub fn kvm_probe() -> Result<()> {
/// Probe the host hypervisor. On Linux that is KVM: open `/dev/kvm`
/// read-write, then check `KVM_GET_API_VERSION` returns 12. On macOS
/// libkrun runs on the Hypervisor.framework, which exists on Apple
/// Silicon Macs and nothing else.
#[cfg(target_os = "linux")]
pub fn hypervisor_probe() -> Result<()> {
use std::os::unix::io::AsRawFd;
let file = std::fs::OpenOptions::new()
.read(true)
@@ -355,10 +397,102 @@ pub fn kvm_probe() -> Result<()> {
Ok(())
}
/// The Hypervisor.framework check, in two steps. `sysctl
/// kern.hv_support` says the machine has the framework; a real
/// `hv_vm_create` says this process may use it — on current macOS the
/// entitlement check inside the framework is what kills most
/// processes, and the denial used to surface only as an opaque
/// `krun_start_enter` EINVAL after a full image build.
#[cfg(target_os = "macos")]
pub fn hypervisor_probe() -> Result<()> {
// libkrun's macOS backend is HVF and arm64-only; there is no
// build for Intel Macs to probe for.
#[cfg(not(target_arch = "aarch64"))]
{
return Err(anyhow::Error::new(Error::HvUnavailable(
std::io::Error::other(
"libkrun ships no macOS backend for Intel Macs; an Apple Silicon host is required",
),
)));
}
#[cfg(target_arch = "aarch64")]
{
let mut supported: libc::c_int = 0;
let mut len = std::mem::size_of_val(&supported);
// Safety: sysctlbyname with a fixed name and an int-sized
// output buffer; the kernel writes only into `supported`.
let rc = unsafe {
libc::sysctlbyname(
c"kern.hv_support".as_ptr(),
(&mut supported as *mut libc::c_int).cast(),
&mut len,
std::ptr::null_mut(),
0,
)
};
if rc != 0 {
let err = std::io::Error::last_os_error();
return Err(anyhow::Error::new(Error::HvUnavailable(err))
.context("reading sysctl kern.hv_support"));
}
if supported != 1 {
return Err(anyhow::Error::new(Error::HvUnavailable(
std::io::Error::other("this Mac reports no Hypervisor.framework support"),
)));
}
hypervisor_entitlement_probe()
}
}
/// Create and destroy a real framework VM. `HV_DENIED` is the denial
/// every macOS packaging step has to design around: the framework
/// grants VM creation only to binaries signed with the
/// `com.apple.security.hypervisor` entitlement, so the fix is a
/// re-sign, not a system change. The framework is dlopen'd by
/// absolute path — ecrou links nothing, and the path is stable across
/// releases; if it cannot be opened at all, the probe defers to the
/// engine instead of inventing a verdict.
#[cfg(target_os = "macos")]
fn hypervisor_entitlement_probe() -> Result<()> {
const HV_DENIED: u32 = 0xfae94007;
let framework =
std::path::Path::new("/System/Library/Frameworks/Hypervisor.framework/Hypervisor");
// Safety: dlopen of a fixed absolute framework path; symbols are
// resolved immediately and the handle lives for this function.
let lib = match unsafe { libloading::Library::new(framework) } {
Ok(lib) => lib,
Err(_) => return Ok(()),
};
let create = symbol!(
&lib,
b"hv_vm_create\0",
unsafe extern "C" fn(*const std::ffi::c_void) -> u32
);
let destroy = symbol!(&lib, b"hv_vm_destroy\0", unsafe extern "C" fn() -> u32);
// Safety: plain calls into the framework; the VM is destroyed
// right away, or never created.
let ret = unsafe { create(std::ptr::null()) };
if ret == 0 {
unsafe { destroy() };
return Ok(());
}
let reason = if ret == HV_DENIED {
"the framework denied the process (HV_DENIED): sign the binary with the \
com.apple.security.hypervisor entitlement, the way packaging does — \
`codesign --force --sign - --entitlements <plist> <binary>`"
} else {
"the framework refused VM creation"
};
Err(anyhow::Error::new(Error::HvUnavailable(
std::io::Error::other(format!("{reason} (hv error 0x{ret:x})")),
)))
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(target_os = "linux")]
#[test]
fn kvm_probe_constant_matches_kvm_abi() {
// KVMIO base 0xAE; KVM_GET_API_VERSION is the first request.
@@ -366,9 +500,10 @@ mod tests {
assert_eq!(KVM_API_VERSION_EXPECTED, 12);
}
#[cfg(target_os = "linux")]
#[test]
fn kvm_probe_reports_unavailability_or_works() {
match kvm_probe() {
match hypervisor_probe() {
Ok(()) => {}
Err(err) => {
let msg = format!("{err:#}");
@@ -378,6 +513,21 @@ mod tests {
}
}
#[cfg(target_os = "macos")]
#[test]
fn hypervisor_probe_reports_unavailability_or_works() {
match hypervisor_probe() {
Ok(()) => {}
Err(err) => {
let msg = format!("{err:#}");
assert!(
msg.contains("Hypervisor.framework"),
"unhelpful probe error: {msg}"
);
}
}
}
#[test]
fn load_fails_with_remediation_or_round_trips() {
match Krun::load() {
@@ -389,14 +539,28 @@ mod tests {
Err(err) => {
let msg = format!("{err:#}");
assert!(msg.contains(SONAME), "bad load error: {msg}");
assert!(
msg.contains("LD_LIBRARY_PATH"),
"remediation missing: {msg}"
);
// The remediation names the platform's install path:
// the loader variable on Linux, the brew tap on macOS.
let hint = if cfg!(target_os = "macos") {
"brew install"
} else {
"LD_LIBRARY_PATH"
};
assert!(msg.contains(hint), "remediation missing: {msg}");
}
}
}
#[test]
fn load_candidates_start_with_the_bare_soname() {
assert_eq!(load_candidates()[0], std::path::PathBuf::from(SONAME));
// The brew keg paths exist only as candidates: dlopen tries
// them in order and they are listed even when absent.
if cfg!(target_os = "macos") {
assert!(load_candidates().len() > 1);
}
}
#[test]
fn ptr_array_is_null_terminated() {
let args = CArgs::build(&["a".to_string(), "b".to_string()]).unwrap();
+3 -3
View File
@@ -17,9 +17,9 @@
//! preparation) and [`vm`] (running a command inside a prepared
//! rootfs). Public functions return [`Result`] carrying the [`Error`]
//! enum, so consumers can match on host-readiness failures
//! ([`Error::KvmUnavailable`], [`Error::ArchNotAvailable`]) versus
//! ordinary failures ([`Error::Other`], which preserves the full context
//! chain).
//! ([`Error::KvmUnavailable`], [`Error::HvUnavailable`],
//! [`Error::ArchNotAvailable`]) versus ordinary failures
//! ([`Error::Other`], which preserves the full context chain).
//!
//! ## Crate split
//!
+65 -8
View File
@@ -10,6 +10,54 @@ use anyhow::{anyhow, Context};
use crate::krun::Krun;
use crate::{Error, Result};
/// The macOS stand-in for `prctl(PR_SET_PDEATHSIG)`: a thread in the
/// child registers a kqueue filter on the parent pid (EVFILT_PROC,
/// NOTE_EXIT) and kills this process with SIGKILL when the parent
/// exits. The kernel fires the filter for an already-dead pid's
/// registration too, and the getppid check catches the window where
/// the parent died before registration; SIGKILL itself cannot be
/// caught, which keeps the child's fate identical to Linux's.
#[cfg(target_os = "macos")]
fn watch_parent_or_die() {
std::thread::spawn(|| {
// Safety: kqueue/kevent on self-owned descriptors; the kevent
// structs are plain value types the kernel fills in.
unsafe {
let kq = libc::kqueue();
if kq < 0 {
return;
}
let change = libc::kevent {
ident: libc::getppid() as usize,
filter: libc::EVFILT_PROC,
flags: libc::EV_ADD,
fflags: libc::NOTE_EXIT,
data: 0,
udata: std::ptr::null_mut(),
};
let n = libc::kevent(kq, &change, 1, std::ptr::null_mut(), 0, std::ptr::null());
// A failed registration or an orphaned ppid means the
// parent is already gone.
if n != 0 || libc::getppid() == 1 {
libc::kill(libc::getpid(), libc::SIGKILL);
}
let mut event: libc::kevent = std::mem::zeroed();
loop {
let n = libc::kevent(kq, std::ptr::null(), 0, &mut event, 1, std::ptr::null());
if n > 0 {
break;
}
if n < 0
&& std::io::Error::last_os_error().kind() != std::io::ErrorKind::Interrupted
{
break;
}
}
libc::kill(libc::getpid(), libc::SIGKILL);
}
});
}
/// Fork the machine process and enter the VM loop inside it. The
/// child only touches libkrun configuration before entering the VM
/// loop, which never returns: a normal payload exit ends the child
@@ -23,7 +71,7 @@ pub(crate) fn spawn_machine(
let child = unsafe { libc::fork() };
if child == 0 {
// Safety: after fork the child runs no threads of its own; the
// prctl/exit pair has no unwinding preconditions.
// kill/exit pair has no unwinding preconditions.
let bail = |msg: String, code: i32| -> ! {
// The console file is the channel the parent streams: a
// boot failure belongs in it, next to whatever the guest
@@ -37,15 +85,24 @@ pub(crate) fn spawn_machine(
eprintln!("Error: {msg}");
unsafe { libc::_exit(code) }
};
// The parent-death contract, per platform: Linux delivers
// SIGKILL through PR_SET_PDEATHSIG; macOS has no prctl, so
// a watcher thread follows the parent through a kqueue and
// kills this process when the parent goes away.
#[cfg(target_os = "linux")]
unsafe {
libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL);
// Headless only: unwire the console input (with a tty on
// fd 0 libkrun puts that terminal into raw mode, which
// fights whoever owns it, and stops a background caller
// with SIGTTOU). /dev/null keeps the console on its plain
// path and still forwards a Ctrl-C sent to the process
// group into the guest. An interactive machine wants the
// opposite: fd 0 is its console input by design.
}
#[cfg(target_os = "macos")]
watch_parent_or_die();
// Headless only: unwire the console input (with a tty on
// fd 0 libkrun puts that terminal into raw mode, which
// fights whoever owns it, and stops a background caller
// with SIGTTOU). /dev/null keeps the console on its plain
// path and still forwards a Ctrl-C sent to the process
// group into the guest. An interactive machine wants the
// opposite: fd 0 is its console input by design.
unsafe {
if !interactive {
let null = libc::open(c"/dev/null".as_ptr(), libc::O_RDWR);
if null >= 0 {
+119
View File
@@ -160,6 +160,125 @@ pub fn map_oci_arch(arch: &str) -> String {
Arch::from_str(arch).oci_name().to_string()
}
// ============================================================================
// External tool lookup
// ============================================================================
/// The Homebrew prefixes the lookup falls back to. Empty off macOS:
/// Homebrew does not exist there and PATH is the whole story.
#[cfg(target_os = "macos")]
pub const BREW_PREFIXES: &[&str] = &["/opt/homebrew", "/usr/local"];
#[cfg(not(target_os = "macos"))]
pub const BREW_PREFIXES: &[&str] = &[];
/// The keg-only formulae the run's external tools come from, in the
/// order they should be preferred. e2fsprogs is keg-only in Homebrew
/// (its sbin never reaches PATH), and the tap ships its own variant
/// built with libarchive, which outranks the core bottle.
#[cfg(target_os = "macos")]
const BREW_TOOL_FORMULAE: &[&str] = &["e2fsprogs-libarchive", "e2fsprogs"];
#[cfg(not(target_os = "macos"))]
const BREW_TOOL_FORMULAE: &[&str] = &[];
/// Locate an external tool the run shells out to: PATH first, then
/// the Homebrew keg directories. Returns the path, so callers spawn a
/// deterministic binary instead of a whatever-came-first `Command`.
pub fn find_tool(name: &str) -> Option<PathBuf> {
if let Some(paths) = std::env::var_os("PATH") {
for dir in std::env::split_paths(&paths) {
let candidate = dir.join(name);
if is_executable(&candidate) {
return Some(candidate);
}
}
}
for prefix in BREW_PREFIXES {
for formula in BREW_TOOL_FORMULAE {
for sub in ["sbin", "bin"] {
let candidate = Path::new(prefix)
.join("opt")
.join(formula)
.join(sub)
.join(name);
if is_executable(&candidate) {
return Some(candidate);
}
}
}
}
None
}
fn is_executable(path: &Path) -> bool {
use std::os::unix::fs::PermissionsExt;
std::fs::metadata(path)
.map(|m| m.is_file() && m.permissions().mode() & 0o111 != 0)
.unwrap_or(false)
}
/// The brew kegs whose dylibs are loaded by bare soname at runtime:
/// libkrunfw (loaded by libkrun) and libarchive (loaded by a
/// libarchive-capable mke2fs). dyld's fallback search never reaches
/// the brew prefix, so these directories have to land in
/// DYLD_FALLBACK_LIBRARY_PATH before any of those loads happen.
#[cfg(target_os = "macos")]
const DYLD_KEGS: &[&str] = &["libkrunfw", "libarchive"];
#[cfg(not(target_os = "macos"))]
const DYLD_KEGS: &[&str] = &[];
/// Directories the libarchive dylib can be dlopen'd from, joined for
/// DYLD_FALLBACK_LIBRARY_PATH. Superseded by dyld_fallback_dirs; kept
/// only until the image builder stops calling it.
#[cfg(target_os = "macos")]
pub fn libarchive_fallback_dirs() -> Option<String> {
let mut dirs = Vec::new();
for prefix in BREW_PREFIXES {
let dir = Path::new(prefix).join("opt/libarchive/lib");
if dir.is_dir() {
dirs.push(dir.as_os_str().to_os_string());
}
}
if dirs.is_empty() {
None
} else {
Some(
std::env::join_paths(dirs)
.ok()?
.to_string_lossy()
.into_owned(),
)
}
}
#[cfg(not(target_os = "macos"))]
pub fn libarchive_fallback_dirs() -> Option<String> {
None
}
/// Directories the bare-soname dlopens can be satisfied from, joined
/// for DYLD_FALLBACK_LIBRARY_PATH, or None when none of them exists.
pub fn dyld_fallback_dirs() -> Option<String> {
let mut dirs = Vec::new();
for prefix in BREW_PREFIXES {
for keg in DYLD_KEGS {
let dir = Path::new(prefix).join("opt").join(keg).join("lib");
if dir.is_dir() {
dirs.push(dir.into_os_string());
}
}
}
if dirs.is_empty() {
None
} else {
Some(
std::env::join_paths(dirs)
.ok()?
.to_string_lossy()
.into_owned(),
)
}
}
#[cfg(test)]
mod tests {
use super::*;
+2 -2
View File
@@ -2,7 +2,7 @@
//! payload with live stdio, propagate its exit code, throw the machine
//! away.
//!
//! The flow is: KVM probe, rootfs prepare (cache-aware download and
//! The flow is: hypervisor probe, rootfs prepare (cache-aware download and
//! extraction), guest init written into the scratch rootfs, libkrun
//! context wired to it, `krun_start_enter` blocking on this thread, then
//! the control file read back for the exit code. The VMM runs inside
@@ -142,7 +142,7 @@ pub fn run_prepared(rootfs: &PreparedRootfs, options: &RunOptions) -> Result<i32
}
}
krun::kvm_probe()?;
krun::hypervisor_probe()?;
let krun = Krun::load()?;
let ram_mib = parse_memory_mib(&options.memory).map_err(Error::Other)?;