diff --git a/web/claw/README.md b/web/claw/README.md index a2aa0b1..0988bb0 100644 --- a/web/claw/README.md +++ b/web/claw/README.md @@ -91,6 +91,14 @@ Two consequences worth knowing before changing this code: like the identity images) and render capped at 240px; code and file cards download on click. Object URLs cache per id, so a replay never refetches. +- **Your images ride along.** The composer's `+` — and an image on the + clipboard — stage as removable chips (`▣1 photo.png 1.2 MB`), the + claw's own budget enforced client-side (8 a message, 5 MiB decoded + each; the draft stays required, the route 422s an empty text). The + sent row renders the local previews at once; the stream's + `user_message` echo carries the stored attachment ids and dedupes + against that row, so a replayed message renders its own thumbnails + through the same bearer-guarded fetch as an attachment card. - **Sessions are created in place.** The sidebar's `+` form posts `POST /v1/sessions` with a client-validated name (`[A-Za-z0-9._-]{1,64}`), switches the chat to the new key, and @@ -116,7 +124,10 @@ from the `addable` kinds the claw serves; each kind's `fields` carry `{name, required}`, `required` marking what the daemon refuses the login without (llama.cpp needs neither of its fields, it serves keyless on a default url); the form demands only those, marks the rest optional, and -takes an optional instance name; unnamed, the id is the kind. A login +takes an optional instance name; unnamed, the id is the kind. Model +options carry a `◉ vision` mark when the live listing knows the model +takes images (`supports_vision: true`); the key's absence is unknown, +never a no, so it renders nothing. A login is `PUT /v1/providers/{id}/credentials`: apply immediately, credentials persist; roster rows are backend instances (`id`, the kind unless the login named it, plus the credential's `source` diff --git a/web/claw/app/app.vue b/web/claw/app/app.vue index ee17380..cf7ed95 100644 --- a/web/claw/app/app.vue +++ b/web/claw/app/app.vue @@ -20,6 +20,7 @@ import type { StreamStatus, } from '~/composables/useClaw' import { useTranscript } from '~/composables/useTranscript' +import type { StagedImage } from '~/composables/useTranscript' const TOKEN_STORE_KEY = 'claw.token' const ACTIVE_STORE_KEY = 'claw.activeKey' @@ -106,18 +107,32 @@ const attemptedAttachments = new Set() // Image cards need their bytes before they can render: fetched as the // card appears, not on any timer. code/file fetch on download instead. +// A replayed question's images fetch the same way — the stream carries +// their attachment ids, never the bytes; locally sent rows hold their +// own object URLs. watch( transcriptItems, (items) => { for (const item of items) { - if (item.kind !== 'attachment') continue - const { id, media, url } = item.attachment - if (media !== 'image' || url !== null || attemptedAttachments.has(id)) continue - attemptedAttachments.add(id) - claw - .attachmentUrl(id) - .then((fetched) => transcript.setAttachmentUrl(id, fetched)) - .catch((err) => noteApiError(err)) + if (item.kind === 'attachment') { + const { id, media, url } = item.attachment + if (media !== 'image' || url !== null || attemptedAttachments.has(id)) continue + attemptedAttachments.add(id) + claw + .attachmentUrl(id) + .then((fetched) => transcript.setAttachmentUrl(id, fetched)) + .catch((err) => noteApiError(err)) + } else if (item.kind === 'sent') { + for (const image of item.images) { + const { id, url } = image + if (id === null || url !== null || attemptedAttachments.has(id)) continue + attemptedAttachments.add(id) + claw + .attachmentUrl(id) + .then((fetched) => transcript.setSentImageUrl(id, fetched)) + .catch((err) => noteApiError(err)) + } + } } }, { deep: true }, @@ -417,13 +432,23 @@ async function removeProvider(id: string): Promise { } } -async function send(text: string): Promise { - transcript.addSent(text) +async function send(message: { text: string; images: StagedImage[] }): Promise { + // The optimistic row renders the staged previews; the stream's + // user_message echo carries the stored ids and dedupes against it. + transcript.addSent( + message.text, + message.images.map(({ media_type, url, name }) => ({ id: null, media_type, url, name })), + ) busy.value = true try { // Fire-and-forget by design: the claw answers 202 and the reply - // arrives on the event stream. - await claw.sendMessage(activeKey.value, text) + // arrives on the event stream. The body carries the images only + // when there are some. + await claw.sendMessage( + activeKey.value, + message.text, + message.images.map(({ media_type, data }) => ({ media_type, data })), + ) apiError.value = null } catch (err) { noteApiError(err) diff --git a/web/claw/app/components/ChatPanel.vue b/web/claw/app/components/ChatPanel.vue index 6728624..dc54a8d 100644 --- a/web/claw/app/components/ChatPanel.vue +++ b/web/claw/app/components/ChatPanel.vue @@ -13,7 +13,7 @@ import { pasteParts, } from '~/composables/useTranscript' import type { ClawSession, ConsentAnswer, StreamStatus } from '~/composables/useClaw' -import type { PastePart, TranscriptItem, TurnBlock } from '~/composables/useTranscript' +import type { PastePart, SentImage, StagedImage, TranscriptItem, TurnBlock } from '~/composables/useTranscript' const props = defineProps<{ items: TranscriptItem[] @@ -31,8 +31,15 @@ const props = defineProps<{ const agentName = computed(() => props.identity?.name?.trim() || 'rhema') +/** One send as the panel hands it up: the text (pastes embedded) and + * the staged images, previews included for the optimistic row. */ +interface SendPayload { + text: string + images: StagedImage[] +} + const emit = defineEmits<{ - send: [text: string] + send: [message: SendPayload] openProviders: [] consentAnswer: [id: string, answer: ConsentAnswer] consentDeny: [id: string] @@ -74,10 +81,15 @@ watch( function submit(): void { const text = draft.value.trim() + // The draft stays the gate: the claw 422s an empty text even with + // images aboard. if (!text || !props.canSend || props.busy) return - emit('send', embedPastes(text, pastes.value)) + emit('send', { text: embedPastes(text, pastes.value), images: stagedImages.value }) + stageGeneration++ draft.value = '' pastes.value = [] + stagedImages.value = [] + imageError.value = null pinned.value = true void nextTick(resizePrompt) } @@ -119,14 +131,22 @@ function onPaste(event: ClipboardEvent): void { if (absorbPaste(text)) event.preventDefault() return } - // A file paste reads as its text when it claims a textual type; the - // rest the composer ignores. - const file = transfer.files[0] - if (!file || !file.type.startsWith('text/')) return - event.preventDefault() - void file.text().then((text) => { - if (!absorbPaste(text)) insertInline(text) - }) + // A file paste: text files read inline or chip, images stage for the + // next send — the same door the (+) button opens; anything else the + // composer still ignores. + let handled = false + for (const file of Array.from(transfer.files)) { + if (file.type.startsWith('text/')) { + handled = true + void file.text().then((pasted) => { + if (!absorbPaste(pasted)) insertInline(pasted) + }) + } else if (mediaTypeOf(file) !== null) { + stageImage(file) + handled = true + } + } + if (handled) event.preventDefault() } /** A long paste becomes one chip; whether it did is the answer. */ @@ -149,6 +169,119 @@ function insertInline(text: string): void { resizePrompt() } +// --- staged images ---------------------------------------------------- +// The (+) button and the clipboard both land here: files held for the +// next send, previewed from object URLs, read to base64 at stage time +// so a bad file surfaces before submit, not inside it. The budget is +// the claw's own (http.rs): eight images, 5 MiB decoded each. + +const MAX_IMAGES = 8 +const MAX_IMAGE_BYTES = 5 * 1024 * 1024 +const IMAGE_TYPES: readonly string[] = ['image/png', 'image/jpeg', 'image/gif', 'image/webp'] +const IMAGE_ACCEPT = IMAGE_TYPES.join(',') + +const stagedImages = ref([]) +const imageError = ref(null) +const fileInput = ref(null) + +/** Bumped on submit: a read still in flight when the draft cleared + * must not stage into a message that already left. */ +let stageGeneration = 0 + +/** The wire media type a file stages as, or null when it is not one + * the claw accepts. The type usually rides the file; browsers that + * cannot name one leave it empty and the extension decides. */ +function mediaTypeOf(file: File): string | null { + if (IMAGE_TYPES.includes(file.type)) return file.type + const byExt: Record = { + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + gif: 'image/gif', + webp: 'image/webp', + } + return byExt[file.name.split('.').pop()?.toLowerCase() ?? ''] ?? null +} + +/** B / KB / MB, one decimal past 1 KB — a chip label, not precision. */ +function formatBytes(size: number): string { + if (size < 1024) return `${size} B` + if (size < 1024 * 1024) return `${(size / 1024).toFixed(1)} KB` + return `${(size / (1024 * 1024)).toFixed(1)} MB` +} + +function pickImages(): void { + fileInput.value?.click() +} + +function onPick(event: Event): void { + const input = event.target as HTMLInputElement + for (const file of Array.from(input.files ?? [])) stageImage(file) + // Cleared so picking the same file again still fires `change`. + input.value = '' +} + +/** Stages one file, or surfaces why not: never a silent drop. */ +function stageImage(file: File): void { + const label = file.name || 'clipboard image' + imageError.value = null + if (stagedImages.value.length >= MAX_IMAGES) { + imageError.value = `${label}: at most ${MAX_IMAGES} images a message` + return + } + const media_type = mediaTypeOf(file) + if (!media_type) { + imageError.value = `${label}: not a png, jpeg, gif, or webp image` + return + } + const generation = stageGeneration + const reader = new FileReader() + reader.onload = () => { + if (generation !== stageGeneration) return + // The data URL's head carries the mime back; the payload after the + // first comma is the bare base64 the body wants. + const dataUrl = typeof reader.result === 'string' ? reader.result : '' + const comma = dataUrl.indexOf(',') + const data = comma === -1 ? '' : dataUrl.slice(comma + 1) + if (data === '') { + imageError.value = `${label}: could not be read` + return + } + // Four base64 characters decode to three bytes — the same budget + // check the claw runs, no decode needed. + if (data.length * 3 / 4 > MAX_IMAGE_BYTES) { + imageError.value = `${label}: over the 5 MiB decoded budget` + return + } + stagedImages.value.push({ media_type, data, url: URL.createObjectURL(file), name: label }) + } + reader.onerror = () => { + if (generation === stageGeneration) imageError.value = `${label}: could not be read` + } + reader.readAsDataURL(file) +} + +function removeImage(index: number): void { + const [gone] = stagedImages.value.splice(index, 1) + if (gone) URL.revokeObjectURL(gone.url) + imageError.value = null +} + +/** A stored image opens the same download path an attachment card + * uses; a local preview has nothing behind it to download. */ +function downloadImage(image: SentImage): void { + if (image.id === null) return + emit('downloadAttachment', image.id, storedImageName(image)) +} + +/** A stored image's download name: the staged file's when the row is + * local, derived from the id when only the claw knows the bytes. */ +function storedImageName(image: SentImage): string { + if (image.name) return image.name + const ext = image.media_type.replace('image/', '') + return `image-${image.id}.${ext === 'jpeg' ? 'jpg' : ext}` +} + /** One render piece of a user message: literal text or a paste chip. */ type MessageSegment = PastePart | { kind: 'chip'; label: string } @@ -224,6 +357,18 @@ function streamTone(status: StreamStatus): 'ok' | 'warn' | 'error' {
you
+
+ +