From 10efe85f381f5930f0e30dee5a61c8b74583c1b9 Mon Sep 17 00:00:00 2001 From: Valentin Haudiquet Date: Thu, 5 Nov 2026 18:52:37 +0100 Subject: [PATCH] feat: the see macro turns a command's output into images There is no native tool for this, and images are shell data like any other, so the surface is a bash function the model pipes into: curl -sL https://example.com/photo.png | see, or see path.png. The name reads as a sentence in the pipe position, keeps working as a plain argument for local files, and is one const in vision.rs if it ever needs to change. Two halves, joined by a staging directory. Provision writes the BASH_ENV init file one non-interactive bash sources; see() execs the rhema binary's hidden __vision mode, which sniffs magic bytes, stages them under a 0700 per-process dir, and says so on stdout. 5 MB per image (the strictest API ceiling), png/jpeg/gif/webp only, and a refusal exits 2 with the reason on stderr so the model reads the limit in-band rather than losing the bytes to a silent drop. Attach runs after every bash command that reaches an exit: ShellTools drains the staging dir into ToolResult.images, a new core field. Bytes never ride stdout, so output caps and truncation cannot eat them, and a staged image attaches even when the command failed. The model staged it, so it belongs in the context. The caps: 5 MB staged, 8 images per result, the rest named in the result's text, because a for loop staging 30 screenshots must not cost a context window. serde keeps the field default, so old checkpoints load unchanged. Anthropic carries native image blocks in the tool_result content array, text first. OpenAI-dialect tool messages (llama.cpp, Copilot, OpenRouter, DeepSeek) and Gemini carry text only, so each withheld image is named in a trailing note the model can act on. The transcript's byte accounting and summaries count the images; the coding-agent config carries the prompt section, and the render adds the section only when wired. Frontends provision only where their bash runs locally: the TUI's session_agent_config, the headless one-shot, the ACP adapter, the claw's daemon, the same gate the MCP wire applies, because a session whose bash runs elsewhere cannot write where the drain reads. The vision wiring lands before the MCP wire so BASH_ENV names the MCP file last and its baked predecessor line reaches the vision init; both functions end up defined. Subagent inner shells inherit the parent's wiring, so a spawned agent keeps the macro in its own commands. SPEC's decision bullet, AGENTS.md's scope line, and the README row land with the feature. --- crates/rhema-claw/SYSTEM.md | 7 ++++ crates/rhema-claw/src/bootstrap.rs | 6 +++ crates/rhema-claw/src/handler.rs | 1 + crates/rhema-claw/src/interactive.rs | 17 +++++++- crates/rhema-claw/src/jobs.rs | 1 + crates/rhema-claw/src/lib.rs | 1 + crates/rhema-claw/src/main.rs | 15 +++++++ crates/rhema-claw/src/mcp.rs | 14 +++---- crates/rhema-claw/src/vision.rs | 61 ++++++++++++++++++++++++++++ 9 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 crates/rhema-claw/src/vision.rs diff --git a/crates/rhema-claw/SYSTEM.md b/crates/rhema-claw/SYSTEM.md index aad4907..aaecb1e 100644 --- a/crates/rhema-claw/SYSTEM.md +++ b/crates/rhema-claw/SYSTEM.md @@ -27,6 +27,13 @@ arguments are the JSON schema a call must satisfy. {% endfor %} {% endif %} +{% if vision %} + +## Viewing images + +{{ vision }} +{% endif %} + ## Memory You keep persistent memory across sessions. memory_save stores what is diff --git a/crates/rhema-claw/src/bootstrap.rs b/crates/rhema-claw/src/bootstrap.rs index 92b4f1f..4087e83 100644 --- a/crates/rhema-claw/src/bootstrap.rs +++ b/crates/rhema-claw/src/bootstrap.rs @@ -212,6 +212,7 @@ pub fn render_system( workspace: &Path, bootstrap: &Bootstrap, tools: &[ToolSpec], + vision: Option<&str>, ) -> Result { let soul = bootstrap.load(BootstrapKind::Soul)?; let user = bootstrap.load(BootstrapKind::User)?; @@ -229,6 +230,7 @@ pub fn render_system( "environment": section("## ENVIRONMENT", environment.as_deref(), GENERATED_AT_BOOT), "onboarding": onboarding, "tools": tools, + "vision": vision, }); render_template(template, &context).map_err(|err| { let line = err @@ -424,6 +426,7 @@ mod tests { Path::new("/ws"), &bootstrap, &tools, + None, ) .expect("renders"); assert!(prompt.contains("## SOUL"), "{prompt}"); @@ -452,6 +455,7 @@ mod tests { Path::new("/ws"), &bootstrap, &[], + None, ) .expect("renders"); assert!( @@ -479,6 +483,7 @@ mod tests { Path::new("/ws"), &bootstrap, &[], + None, ) .expect("renders"); assert!(prompt.contains(content), "{prompt}"); @@ -510,6 +515,7 @@ mod tests { Path::new("/ws"), &bootstrap, &tools, + None, ) .expect("renders"); assert!(prompt.contains("- my_tool | its purpose | "), "{prompt}"); diff --git a/crates/rhema-claw/src/handler.rs b/crates/rhema-claw/src/handler.rs index a1acb32..12ec4a4 100644 --- a/crates/rhema-claw/src/handler.rs +++ b/crates/rhema-claw/src/handler.rs @@ -246,6 +246,7 @@ impl ClawHandler { .transcript .then(rhema_transcript::TranscriptSettings::default); config.shell_env = self.tool_env.clone(); + crate::vision::apply(&mut config); crate::mcp::apply(&mut config); config } diff --git a/crates/rhema-claw/src/interactive.rs b/crates/rhema-claw/src/interactive.rs index 52911df..27cc9d3 100644 --- a/crates/rhema-claw/src/interactive.rs +++ b/crates/rhema-claw/src/interactive.rs @@ -222,6 +222,10 @@ pub struct InteractiveHandler { /// decorator refreshes it per request, `zoom` reads it, and it /// lives here — the agent rebuilds per dispatch, the log must not. transcript_mirror: Option>>, + /// The wired [`vision`](rhema_agent_tools::vision) section for the + /// prompt, when provisioning succeeded — the same wiring whose env + /// map lights the shell's inbox up. + vision: Option, } impl InteractiveHandler { @@ -247,7 +251,15 @@ impl InteractiveHandler { transcript: bool, ) -> Self { let workspace = workspace.into(); - let tools = Arc::new(ShellTools::new(&workspace)); + // The vision wiring rides the shell's env: `see` exists in the + // chat's commands, and its staged images attach to their + // results. Local shells only — the interactive handler's + // executor is this host's by construction. + let vision = rhema_agent_tools::vision::wire(std::env::var_os("BASH_ENV")); + let tools = Arc::new(match vision { + Some(wiring) => ShellTools::with_env(&workspace, &wiring.shell_env), + None => ShellTools::new(&workspace), + }); tools.install_approval(Arc::new(AllowAll)); let bootstrap_resolver = crate::bootstrap::registry(bootstrap.clone()).resolver_for(&Allowlist::permissive()); @@ -268,6 +280,7 @@ impl InteractiveHandler { rhema_transcript::TranscriptSettings::default(), ))) }), + vision: vision.map(|wiring| wiring.section.to_owned()), } } @@ -302,6 +315,7 @@ impl InteractiveHandler { &self.workspace, &self.bootstrap, &resolver.specs(), + self.vision.as_deref(), ) .map_err(|err| format!("cannot render the bootstrap system prompt: {err}"))?; Agent::builder() @@ -1063,6 +1077,7 @@ mod tests { Path::new("/state/claw/workspaces/interactive"), &handler.bootstrap, &specs, + None, ) .expect("renders"); assert!( diff --git a/crates/rhema-claw/src/jobs.rs b/crates/rhema-claw/src/jobs.rs index 0f02a18..e275f23 100644 --- a/crates/rhema-claw/src/jobs.rs +++ b/crates/rhema-claw/src/jobs.rs @@ -207,6 +207,7 @@ impl CodingJobHandler { config.transcript = self .transcript .then(rhema_transcript::TranscriptSettings::default); + crate::vision::apply(&mut config); crate::mcp::apply(&mut config); let stashed = self .store diff --git a/crates/rhema-claw/src/lib.rs b/crates/rhema-claw/src/lib.rs index bc03fb8..f95ba03 100644 --- a/crates/rhema-claw/src/lib.rs +++ b/crates/rhema-claw/src/lib.rs @@ -92,6 +92,7 @@ pub mod task_source; pub mod task_store; pub mod task_tools; pub mod turn_store; +pub(crate) mod vision; pub mod watch_store; pub mod watch_tools; diff --git a/crates/rhema-claw/src/main.rs b/crates/rhema-claw/src/main.rs index c4c2184..aabb6ec 100644 --- a/crates/rhema-claw/src/main.rs +++ b/crates/rhema-claw/src/main.rs @@ -50,6 +50,15 @@ enum Command { #[arg(allow_hyphen_values = true, trailing_var_arg = true)] args: Vec, }, + /// The vision CLI the agent shell's `see` macro execs: stages the + /// piped bytes (or the named files) into the session's staging + /// directory. Outside a wired shell it exits 4 with a hint. + #[command(name = "__vision")] + Vision { + /// The CLI's own argv, passed through untouched. + #[arg(allow_hyphen_values = true, trailing_var_arg = true)] + args: Vec, + }, } #[tokio::main] @@ -63,6 +72,12 @@ async fn main() { argv.extend(rest.iter().cloned()); std::process::exit(rhema_mcp::cli::run(&argv)); } + if let Some(Command::Vision { args: rest }) = &args.command { + let mut argv = Vec::with_capacity(rest.len() + 1); + argv.push(std::ffi::OsString::from(rhema_agent_tools::vision::MODE)); + argv.extend(rest.iter().cloned()); + std::process::exit(rhema_agent_tools::vision::cli_run(&argv)); + } if let Err(err) = run(args).await { eprintln!("error: {err}"); let mut source: &dyn std::error::Error = &err; diff --git a/crates/rhema-claw/src/mcp.rs b/crates/rhema-claw/src/mcp.rs index 6f352ba..9020b6c 100644 --- a/crates/rhema-claw/src/mcp.rs +++ b/crates/rhema-claw/src/mcp.rs @@ -34,14 +34,12 @@ fn wired() -> Option<&'static Wired> { } let dir = std::env::temp_dir().join(format!("rhema-claw-mcp-{}", std::process::id())); std::fs::create_dir_all(&dir).ok()?; - let wiring = rhema_mcp::wire( - &mcp, - &claw.disabled_mcp_servers, - &dir, - "claw", - std::env::var_os("BASH_ENV"), - ) - .ok()?; + // The init chain: the vision wiring first, this one + // second — `BASH_ENV` names ours, whose inheritance guard + // sources the vision file, which sources the process's own. + let inherited = crate::vision::init_path().or_else(|| std::env::var_os("BASH_ENV")); + let wiring = + rhema_mcp::wire(&mcp, &claw.disabled_mcp_servers, &dir, "claw", inherited).ok()?; for notice in &wiring.notices { eprintln!("· {notice}"); } diff --git a/crates/rhema-claw/src/vision.rs b/crates/rhema-claw/src/vision.rs new file mode 100644 index 0000000..45ff8e0 --- /dev/null +++ b/crates/rhema-claw/src/vision.rs @@ -0,0 +1,61 @@ +//! The claw's vision wiring: one provision for the daemon process, its +//! env merged into every coding-agent session the claw runs — chat +//! turns and `coding_agent` jobs alike. There is no config gate: the +//! `see` macro is a shell affordance, not a server, so it wires +//! whenever the daemon's agents run local shells. `apply` must run +//! before [`crate::mcp::apply`]: the MCP init file bakes this one in +//! as its literal predecessor, so `BASH_ENV` has to name the MCP file +//! last. + +#![forbid(unsafe_code)] + +use std::sync::{Arc, OnceLock}; + +use rhema_coding_agent::CodingAgentConfig; + +static WIRED: OnceLock>> = OnceLock::new(); + +/// The daemon's wiring, established at first use and kept for the +/// process's life. `None` only when provisioning failed, cached like +/// every answer this static gives. +pub(crate) fn wired() -> Option<&'static Arc> { + WIRED + .get_or_init(|| { + rhema_agent_tools::vision::wire(std::env::var_os("BASH_ENV")).map(Arc::clone) + }) + .as_ref() +} + +/// The init file a later provisioner chains through — `None` when the +/// vision wiring itself failed, in which case the chain falls back to +/// the process's own `BASH_ENV`. +pub(crate) fn init_path() -> Option { + wired().map(|wiring| std::ffi::OsString::from(wiring.init_path.display().to_string())) +} + +/// Layers the wiring onto one agent config: the shell env map (the +/// init file, the binary, the staging directory) and the prompt's +/// vision section. A frontend without wiring changes nothing. +pub(crate) fn apply(config: &mut CodingAgentConfig) { + let Some(wired) = wired() else { + return; + }; + config.shell_env.extend(wired.shell_env.clone()); + config.vision = Some(wired.section.to_owned()); +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A first `apply` is indistinguishable from a second: the map's + /// values are constants, so extending twice is the same map. + #[test] + fn apply_twice_is_apply_once() { + let mut config = CodingAgentConfig::new("m", std::path::Path::new("/tmp")); + apply(&mut config); + let once = config.shell_env.clone(); + apply(&mut config); + assert_eq!(config.shell_env, once); + } +}