feat(web): attach images from the composer
The (+) door and the clipboard door both stage files as removable chips, the claw's own budget enforced client-side (eight a message, 5 MiB decoded each, errors surfaced, never silent drops). The sent row renders local previews at once; the stream's user_message echo carries the stored ids and dedupes against it, so a replayed message renders its thumbnails through the same bearer-guarded fetch as an attachment card. Model options mark vision when the listing knows.
This commit is contained in:
1 parent
cfb349508f
commit
6efb69331d
6 files changed
+396
-39
No files matched your search
@@ -41,9 +41,24 @@ export interface ClawAttachment {
|
||||
caption: string | null
|
||||
}
|
||||
|
||||
/** One image a sent message carries, as the `POST .../messages` body
|
||||
* takes it: the full MIME type plus bare base64 (http.rs `WireImage`). */
|
||||
export interface OutboundImage {
|
||||
media_type: string
|
||||
data: string
|
||||
}
|
||||
|
||||
/** One image a `user_message` event carries: the attachment id the
|
||||
* bytes live under (fetch them via `attachmentUrl`) plus the MIME type
|
||||
* — megabytes never ride the stream (hub.rs `UserImage`). */
|
||||
export interface UserImage {
|
||||
id: string
|
||||
media_type: string
|
||||
}
|
||||
|
||||
/** One turn event, exactly `hub.rs`'s `TurnEvent` (serde type tag, snake case). */
|
||||
export type TurnEvent =
|
||||
| { type: 'user_message'; text: string }
|
||||
| { type: 'user_message'; text: string; images: UserImage[] }
|
||||
| { type: 'thinking_delta'; text: string }
|
||||
| { type: 'text_delta'; text: string }
|
||||
| { type: 'tool_start'; name: string }
|
||||
@@ -78,6 +93,9 @@ export interface ModelCard {
|
||||
display_name: string
|
||||
description: string
|
||||
input_token_limit?: number | null
|
||||
/** Whether the model takes image input, when the adapter knows:
|
||||
* absent means unknown — no verdict, never a `false`. */
|
||||
supports_vision?: boolean
|
||||
}
|
||||
|
||||
/** One account of `GET /v1/accounts` (rhema-identity's `Account`):
|
||||
@@ -308,8 +326,8 @@ export interface ClawApi {
|
||||
* a plain src will not do). Cached per id: attachments are immutable,
|
||||
* so a replayed event never refetches. */
|
||||
attachmentUrl(id: string): Promise<string>
|
||||
/** `POST /v1/sessions/{key}/messages` — fire-and-forget: the claw answers 202 and the reply arrives on the event stream. */
|
||||
sendMessage(key: string, text: string): Promise<void>
|
||||
/** `POST /v1/sessions/{key}/messages` — fire-and-forget: the claw answers 202 and the reply arrives on the event stream. Images ride the body only when there are some; without them the body is the old `{"text"}` shape byte-for-byte. The route 422s an empty text even with images aboard. */
|
||||
sendMessage(key: string, text: string, images?: OutboundImage[]): Promise<void>
|
||||
/** `GET /v1/providers` — the constructible backends and the active pair. */
|
||||
providers(): Promise<ProvidersState>
|
||||
/** `PUT /v1/providers/active` — switch the backend, persisted claw-side. Answers the new active pair only. */
|
||||
@@ -410,11 +428,11 @@ export function useClaw(token: Ref<string | null>): ClawApi {
|
||||
})
|
||||
}
|
||||
|
||||
async function sendMessage(key: string, text: string): Promise<void> {
|
||||
async function sendMessage(key: string, text: string, images?: OutboundImage[]): Promise<void> {
|
||||
await send(`/v1/sessions/${encodeURIComponent(key)}/messages`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', ...authHeaders(token.value) },
|
||||
body: JSON.stringify({ text }),
|
||||
body: JSON.stringify(images?.length ? { text, images } : { text }),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -649,12 +667,33 @@ async function readFrames(
|
||||
}
|
||||
}
|
||||
|
||||
/** The images a `user_message` carries: rows recorded before images
|
||||
* rode the transcript carry no field at all, and an entry without an
|
||||
* id and an image media type is not one — both fold to fewer (or no)
|
||||
* images rather than a guessed shape. */
|
||||
function parseUserImages(raw: unknown): UserImage[] {
|
||||
if (!Array.isArray(raw)) return []
|
||||
const images: UserImage[] = []
|
||||
for (const entry of raw) {
|
||||
const image = entry as Record<string, unknown>
|
||||
if (typeof image.id !== 'string' || typeof image.media_type !== 'string') continue
|
||||
if (!image.media_type.startsWith('image/')) continue
|
||||
images.push({ id: image.id, media_type: image.media_type })
|
||||
}
|
||||
return images
|
||||
}
|
||||
|
||||
/** Parses one `data:` payload into a `TurnEvent`; unknown shapes are dropped, not guessed. */
|
||||
function parseTurnEvent(data: string): TurnEvent | null {
|
||||
try {
|
||||
const raw = JSON.parse(data) as Record<string, unknown>
|
||||
switch (raw.type) {
|
||||
case 'user_message':
|
||||
return {
|
||||
type: 'user_message',
|
||||
text: String(raw.text ?? ''),
|
||||
images: parseUserImages(raw.images),
|
||||
}
|
||||
case 'thinking_delta':
|
||||
case 'text_delta':
|
||||
return { type: raw.type, text: String(raw.text ?? '') }
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
// mirror when the stream reopens and folds what arrives, never the other
|
||||
// way around.
|
||||
import { ref } from 'vue'
|
||||
import type { AttachmentMedia, ConsentKind, TurnEvent } from './useClaw'
|
||||
import type { AttachmentMedia, ConsentKind, OutboundImage, TurnEvent } from './useClaw'
|
||||
|
||||
/** One tool call inside a turn: started, then completed by its result. */
|
||||
export interface ToolRow {
|
||||
@@ -63,9 +63,33 @@ export interface AttachmentCard {
|
||||
url: string | null
|
||||
}
|
||||
|
||||
/** One image a user message carried, as the transcript renders it. A
|
||||
* locally sent row holds the browser's own object URL — no fetch, no
|
||||
* id; a replayed row holds the attachment id the claw stored and gets
|
||||
* its bytes bearer-guarded, like an attachment card. */
|
||||
export interface SentImage {
|
||||
/** The attachment id behind a replayed row; null for a local one. */
|
||||
id: string | null
|
||||
media_type: string
|
||||
/** The object URL once resolved: local at stage time, fetched for a replay. */
|
||||
url: string | null
|
||||
/** The file's name when the browser knew one; a replay has only an id. */
|
||||
name: string | null
|
||||
}
|
||||
|
||||
/** One image staged in the composer, as the send path hands it up: the
|
||||
* wire payload plus the local bits the optimistic row needs and the
|
||||
* claw never sees. */
|
||||
export interface StagedImage extends OutboundImage {
|
||||
/** The local preview URL the optimistic row renders. */
|
||||
url: string
|
||||
/** The file's name, as the chip and the sent row label it. */
|
||||
name: string
|
||||
}
|
||||
|
||||
/** A transcript line: a message the local user sent, or one streamed turn. */
|
||||
export type TranscriptItem =
|
||||
| { kind: 'sent'; id: number; text: string }
|
||||
| { kind: 'sent'; id: number; text: string; images: SentImage[] }
|
||||
| { kind: 'turn'; id: number; turn: TurnBlock }
|
||||
| { kind: 'consent'; id: number; consent: ConsentCard }
|
||||
| { kind: 'attachment'; id: number; attachment: AttachmentCard }
|
||||
@@ -169,10 +193,22 @@ export function useTranscript() {
|
||||
case 'user_message': {
|
||||
// The replayed question. A live echo of a message this surface
|
||||
// just sent arrives after the optimistic `sent` row: identical
|
||||
// text as the last item means the row already stands in for it.
|
||||
// text and the same image count as the last item means the row
|
||||
// already stands in for it, previews and all.
|
||||
const last = items.value[items.value.length - 1]
|
||||
if (last?.kind === 'sent' && last.text === event.text) return
|
||||
items.value.push({ kind: 'sent', id: nextId++, text: event.text })
|
||||
if (
|
||||
last?.kind === 'sent' &&
|
||||
last.text === event.text &&
|
||||
last.images.length === event.images.length
|
||||
) {
|
||||
return
|
||||
}
|
||||
items.value.push({
|
||||
kind: 'sent',
|
||||
id: nextId++,
|
||||
text: event.text,
|
||||
images: event.images.map((image) => ({ ...image, url: null, name: null })),
|
||||
})
|
||||
break
|
||||
}
|
||||
case 'thinking_delta': {
|
||||
@@ -280,9 +316,11 @@ export function useTranscript() {
|
||||
}
|
||||
}
|
||||
|
||||
/** A locally sent message, shown immediately; the reply streams after. */
|
||||
function addSent(text: string): void {
|
||||
items.value.push({ kind: 'sent', id: nextId++, text })
|
||||
/** A locally sent message, shown immediately; the reply streams after.
|
||||
* Its images render from the staged previews — the claw stores them
|
||||
* and the stream's echo dedupes against this row. */
|
||||
function addSent(text: string, images: SentImage[] = []): void {
|
||||
items.value.push({ kind: 'sent', id: nextId++, text, images })
|
||||
}
|
||||
|
||||
/** A consent card resolved: flipped in place, or ignored when no
|
||||
@@ -324,6 +362,21 @@ export function useTranscript() {
|
||||
}
|
||||
}
|
||||
|
||||
/** A replayed sent image's object URL landed; stored on its row. The
|
||||
* useClaw cache keeps a reconnect's replay from refetching the bytes.
|
||||
* A user message is always a top-level row — it never rides a live
|
||||
* turn — so the scan stays over `items`. */
|
||||
function setSentImageUrl(id: string, url: string): void {
|
||||
for (const item of items.value) {
|
||||
if (item.kind !== 'sent') continue
|
||||
const image = item.images.find((candidate) => candidate.id === id)
|
||||
if (image) {
|
||||
image.url = url
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether a card for this consent id is already on the mirror — a
|
||||
* replayed request never stacks a second card beside the first. */
|
||||
function findConsent(id: string): boolean {
|
||||
@@ -346,5 +399,5 @@ export function useTranscript() {
|
||||
live.value = null
|
||||
}
|
||||
|
||||
return { items, live, applyEvent, addSent, resolveConsent, setAttachmentUrl, reset }
|
||||
return { items, live, applyEvent, addSent, resolveConsent, setAttachmentUrl, setSentImageUrl, reset }
|
||||
}
|
||||
Reference in new issue
Block a user