diff --git a/android/README.md b/android/README.md index 962a922..af45568 100644 --- a/android/README.md +++ b/android/README.md @@ -20,6 +20,10 @@ manages the device fleet the claw keeps. backend instances (`GET /v1/providers`), each one's live model listing, and the active pair staged as a draft behind an explicit save (`PUT /v1/providers/active`; it applies from the claw's next turn). + Model options carry the listing's vision verdict the way the web and + the TUI spell it — `◉ vision` when the adapter knows the model takes + images, a bare `no vision` when it knows it does not, nothing when the + key is absent (unknown is no verdict, never a `false`). The **Diagnostics** page answers "why is the fleet row stale" from the phone's side: the three WorkManager jobs' live state (an empty info list — a cancelled schedule — is the loud answer), the two clocks @@ -53,7 +57,19 @@ manages the device fleet the claw keeps. types skipped). The reader's own messages render as right-anchored bubbles (elevated surface, accent bar on the trailing edge); the agent's turns keep the avatar on the left. The - composer POSTs `{"text"}`; the claw's documented dance is preserved: + composer's (+) door stages images for the next send — a photo-picker + pick or a camera shot written straight into a FileProvider-mapped + cache uri (no CAMERA permission) — each a removable `▣N name size` + chip with a downscaled thumbnail above the input row. The claw's own + budget is enforced client-side with visible inline errors, never + silent drops: at most eight images a message, 5 MiB decoded each, + png/jpeg/gif/webp only (a heic pick refuses rather than being sent), + and the text stays required — the claw 422s an empty text even with + images aboard. Send reads the staged files back and POSTs bare base64 + in the body's `images` (omitted when there are none, so the old + `{"text"}` shape stays byte-stable); a failed send restores the draft + and the chips. The composer POSTs `{"text"}` plus those images; the + claw's documented dance is preserved: **subscribe first, then POST**, the reply arrives on the stream. Thinking deltas render dim and collapsible, tool rows show name/summary with error coloring (bash-like tools render as shell lines instead: dim `$` prefix, @@ -63,7 +79,15 @@ manages the device fleet the claw keeps. preview for small (≤ 2 MB) images, and a download-and-open row (bytes from `GET /v1/attachments/{id}` land in the app cache and open through a FileProvider `ACTION_VIEW`, the only way another app can read a private - cache file). The agent's `consent_request` questions (a secret, or an + cache file). Sent messages with images render their thumbnails in the + right-anchored bubble: the optimistic row previews the staged files at + once, and a replayed `user_message` (the stream carries attachment + ids, never bytes) fetches through the same bearer-guarded + `GET /v1/attachments/{id}` as an attachment card, decoded downscaled + and capped at the card's 240 dp. The stream's echo of a message this + surface just sent dedupes against the optimistic row (same text, same + image count), so the local previews stand. The agent's + `consent_request` questions (a secret, or an approval of a phone control action) render as cards answered in place — `POST`/`DELETE /v1/consents/{id}` — and `consent_resolved` settles them (a reconnect's replay rebuilds the settled states). The streaming reply @@ -190,7 +214,7 @@ All under one base URL; `Authorization: Bearer ` on `/v1`, never on | `GET /v1/providers/{id}/models` | an instance's live model listing | | `PUT /v1/providers/active` | switch the backend instance, persisted | | `GET /v1/sessions` | session list | -| `POST /v1/sessions/{key}/messages` | `{"text"}` → 202; reply via stream | +| `POST /v1/sessions/{key}/messages` | `{"text"}`, optional `images` (≤ 8, ≤ 5 MiB decoded each, bare base64) → 202; reply via stream | | `GET /v1/sessions/{key}/events` | SSE, replay-then-live: `user_message`, `thinking_delta`, `text_delta`, `tool_start`, `tool_result`, `turn_done`, `attachment`, `consent_request`, `consent_resolved`. `?since=N` skips the replay's oldest N events — the transcript cache's cursor | | `GET /v1/attachments/{id}` | an attachment event's bytes (bearer) | | `GET /v1/consents` | pending consent questions, oldest first | diff --git a/android/app/src/main/java/rs/rhema/phone/chat/ChatController.kt b/android/app/src/main/java/rs/rhema/phone/chat/ChatController.kt index 6d3a95c..21c45c2 100644 --- a/android/app/src/main/java/rs/rhema/phone/chat/ChatController.kt +++ b/android/app/src/main/java/rs/rhema/phone/chat/ChatController.kt @@ -1,5 +1,6 @@ package rs.rhema.phone.chat +import android.util.Base64 import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.setValue @@ -9,14 +10,25 @@ import rs.rhema.phone.data.SessionCache import rs.rhema.phone.net.ClawApi import rs.rhema.phone.net.ClawEvent import rs.rhema.phone.net.ClawResult +import rs.rhema.phone.net.OutboundImage import rs.rhema.phone.net.SseFrame import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import kotlinx.coroutines.NonCancellable +import java.io.File +import java.io.IOException + +// The claw's own image budget (POST .../messages): eight images a +// message, 5 MiB decoded each. The composer enforces it at stage time; +// send() checks again so a stale staged file cannot smuggle an +// oversized body past it. +const val MAX_STAGED_IMAGES = 8 +const val MAX_IMAGE_BYTES = 5L * 1024 * 1024 // Holds the live transcript of one open chat screen. The transcript is // seeded from the session cache before the first frame and kept in the @@ -34,7 +46,20 @@ class ChatController( val id: Long } - data class UserMessage(override val id: Long, val text: String) : Item + // One image a user row renders: the staged file behind a message + // this surface just sent (previews locally, no id yet), or the + // attachment id a replayed row names (bytes live at the claw). + data class SentImage( + val attachmentId: String?, + val file: File?, + val mediaType: String, + ) + + data class UserMessage( + override val id: Long, + val text: String, + val images: List = emptyList(), + ) : Item data class Thinking(override val id: Long, val text: String) : Item data class ToolCall( override val id: Long, @@ -158,22 +183,57 @@ class ChatController( streamJob = null } + // One image staged for the next send: the file the optimistic row + // previews, the wire type the POST body carries, and its decoded + // size for the budget's last look. + data class StagedImage(val file: File, val mediaType: String, val sizeBytes: Long) + // The stream is already collecting by the time the user can press send, - // so this preserves the claw's subscribe-then-post order. A failed POST - // restores the draft in the UI. The turn's user_message event is the + // so this preserves the claw's subscribe-then-post order. Images are + // read and encoded here — the send path's own budget look (the composer + // already enforced it at stage time); a violation or an unreadable file + // fails the send with the error surfaced, never a silent drop. A failed + // POST restores the draft in the UI. The turn's user_message event is the // transcript's own record of the question — replayed on a reload, and // echoed live right about when this returns — so the local row is only // appended when the echo has not already landed. - suspend fun send(text: String): Boolean { + suspend fun send(text: String, staged: List = emptyList()): Boolean { + if (staged.size > MAX_STAGED_IMAGES) { + sendError = "over the $MAX_STAGED_IMAGES-image budget" + return false + } sending = true sendError = null - val result = api.postMessage(sessionKey, text) + val outbound = mutableListOf() + for (image in staged) { + try { + val bytes = withContext(Dispatchers.IO) { image.file.readBytes() } + if (bytes.size > MAX_IMAGE_BYTES) { + sending = false + sendError = "image over the 5 MiB decoded budget" + return false + } + outbound += OutboundImage( + image.mediaType, + Base64.encodeToString(bytes, Base64.NO_WRAP), + ) + } catch (e: IOException) { + sending = false + sendError = "image could not be read" + return false + } + } + val result = api.postMessage(sessionKey, text, outbound) sending = false return when (result) { is ClawResult.Ok -> { val last = items.lastOrNull() - if (!(last is UserMessage && last.text == text)) { - items = items + UserMessage(newId(), text) + if (!(last is UserMessage && last.text == text && last.images.size == staged.size)) { + items = items + UserMessage( + newId(), + text, + staged.map { SentImage(attachmentId = null, file = it.file, mediaType = it.mediaType) }, + ) } true } @@ -200,10 +260,15 @@ class ChatController( is ClawEvent.UserMessage -> { // The recorded question — replayed on reload, echoed live // for a message this surface just sent. An identical last - // row is that echo's stand-in; skip it. + // row — same text, same image count — is that echo's + // stand-in; skip it, previews and all. val last = list.lastOrNull() - if (!(last is UserMessage && last.text == event.text)) { - list = list + UserMessage(newId(), event.text) + if (!(last is UserMessage && last.text == event.text && last.images.size == event.images.size)) { + list = list + UserMessage( + newId(), + event.text, + event.images.map { SentImage(attachmentId = it.id, file = null, mediaType = it.mediaType) }, + ) } } is ClawEvent.ThinkingDelta -> { diff --git a/android/app/src/main/java/rs/rhema/phone/net/ClawApi.kt b/android/app/src/main/java/rs/rhema/phone/net/ClawApi.kt index 3c3edc6..4e2b893 100644 --- a/android/app/src/main/java/rs/rhema/phone/net/ClawApi.kt +++ b/android/app/src/main/java/rs/rhema/phone/net/ClawApi.kt @@ -102,10 +102,16 @@ class ClawApi( // The claw's documented dance: subscribe to the event stream first, then // POST; the reply arrives on the stream, not in the POST response (202). - suspend fun postMessage(sessionKey: String, text: String): ClawResult { + // Staged images ride the body only when there are some (the request + // type's default keeps the old shape byte-stable otherwise). + suspend fun postMessage( + sessionKey: String, + text: String, + images: List = emptyList(), + ): ClawResult { val builder = v1("sessions", sessionKey, "messages") ?: return ClawResult.Network("no base URL configured") - val body = json.encodeToString(MessageRequest(text)).toRequestBody(jsonBody) + val body = json.encodeToString(MessageRequest(text, images)).toRequestBody(jsonBody) return execute(builder.post(body).build()) { } } diff --git a/android/app/src/main/java/rs/rhema/phone/net/Models.kt b/android/app/src/main/java/rs/rhema/phone/net/Models.kt index a044611..c20fed4 100644 --- a/android/app/src/main/java/rs/rhema/phone/net/Models.kt +++ b/android/app/src/main/java/rs/rhema/phone/net/Models.kt @@ -20,9 +20,20 @@ data class SessionInfo( @SerialName("last_outcome") val lastOutcome: JsonElement? = null, ) -// POST /v1/sessions/{key}/messages +// POST /v1/sessions/{key}/messages. The Json instance skips defaults, +// so an empty image list keeps the old {"text"} body byte-for-byte. +// Text stays required: the claw 422s an empty text even with images. @Serializable -data class MessageRequest(val text: String) +data class MessageRequest(val text: String, val images: List = emptyList()) + +// One image in the POST body: the full MIME type plus bare base64. The +// claw's budget — at most eight, 5 MiB decoded each, png/jpeg/gif/webp +// only — is enforced client-side at stage time (the composer). +@Serializable +data class OutboundImage( + @SerialName("media_type") val mediaType: String, + val data: String, +) // GET /v1/providers — the backend instances and the active pair. Each // row is one instance, named by its id: the kind unless a login named @@ -46,6 +57,9 @@ data class ModelCard( val id: String, @SerialName("display_name") val displayName: String? = null, val description: String? = null, + // Whether the model takes image input, when the adapter knows: + // absent means unknown — no verdict, never a false. + @SerialName("supports_vision") val supportsVision: Boolean? = null, ) @Serializable @@ -97,6 +111,12 @@ data class RhemaNotification( @SerialName("created_at") val createdAt: Long, ) +// One image a user_message names: the attachment id the bytes live +// under (GET /v1/attachments/{id}, bearer-guarded) plus the MIME type — +// megabytes never ride the stream. +@Serializable +data class UserImage(val id: String, @SerialName("media_type") val mediaType: String) + // Frames of GET /v1/sessions/{key}/events; "type" is the discriminator. // The stream opens with the session's recorded turns replayed before the // live feed, so a screen (re)builds its transcript from the replay. @@ -106,7 +126,12 @@ data class RhemaNotification( sealed interface ClawEvent { @Serializable @SerialName("user_message") - data class UserMessage(val text: String = "") : ClawEvent + // Rows recorded before images rode the stream carry no field at + // all; absent folds to none, never a guessed shape. + data class UserMessage( + val text: String = "", + val images: List = emptyList(), + ) : ClawEvent @Serializable @SerialName("thinking_delta") diff --git a/android/app/src/main/java/rs/rhema/phone/ui/ChatScreen.kt b/android/app/src/main/java/rs/rhema/phone/ui/ChatScreen.kt index 09a0dbf..c8462c2 100644 --- a/android/app/src/main/java/rs/rhema/phone/ui/ChatScreen.kt +++ b/android/app/src/main/java/rs/rhema/phone/ui/ChatScreen.kt @@ -6,9 +6,16 @@ import android.content.Intent import android.graphics.BitmapFactory import android.net.Uri import androidx.activity.compose.BackHandler +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.PickVisualMediaRequest +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.BorderStroke import androidx.compose.foundation.Image import androidx.compose.foundation.background +import androidx.compose.foundation.border import androidx.compose.foundation.clickable +import androidx.compose.foundation.horizontalScroll +import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column @@ -21,12 +28,16 @@ import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.IntrinsicSize import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.widthIn import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.foundation.lazy.items import androidx.compose.foundation.shape.CircleShape import androidx.compose.material.icons.Icons import androidx.compose.material.icons.automirrored.filled.ArrowBack import androidx.compose.material.icons.automirrored.filled.Send +import androidx.compose.material.icons.filled.Add +import androidx.compose.material.icons.filled.Close +import androidx.compose.material3.AlertDialog import androidx.compose.material3.Button import androidx.compose.material3.HorizontalDivider import androidx.compose.material3.Icon @@ -49,6 +60,7 @@ import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip import androidx.compose.ui.graphics.ImageBitmap import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.layout.ContentScale @@ -60,8 +72,11 @@ import androidx.compose.ui.text.input.PasswordVisualTransformation import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp import androidx.core.content.FileProvider +import androidx.core.net.toUri import rs.rhema.phone.R import rs.rhema.phone.chat.ChatController +import rs.rhema.phone.chat.MAX_IMAGE_BYTES +import rs.rhema.phone.chat.MAX_STAGED_IMAGES import rs.rhema.phone.data.AgentIdentityCache import rs.rhema.phone.di.AppContainer import rs.rhema.phone.net.ClawResult @@ -70,6 +85,8 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import java.io.File import java.io.IOException +import java.util.Locale +import android.provider.OpenableColumns // The transcript screen, the design's chat panel as a phone screen: // avatar rows per turn (the tool cards with their one-sided accent @@ -89,6 +106,65 @@ fun ChatScreen(container: AppContainer, sessionKey: String, onBack: () -> Unit) var draft by rememberSaveable { mutableStateOf("") } + // Images staged for the next send, held as files in the app cache: + // the picker's read grant dies with the activity result, while the + // send reads the staged file whenever the message leaves. + var staged by remember { mutableStateOf>(emptyList()) } + var stageError by remember { mutableStateOf(null) } + var attachOpen by remember { mutableStateOf(false) } + // The camera's content:// target, remembered so the result callback + // still knows what was written even after a configuration change. + var cameraTarget by rememberSaveable { mutableStateOf(null) } + val context = LocalContext.current + // Labels resolve in composition, not at callback time: a lookup + // through LocalContext.current freezes the first composition's + // locale (the same rule the attachment card's templates follow). + val cameraLabel = stringResource(R.string.chat_attach_camera) + val cameraShotName = stringResource(R.string.chat_camera_shot_name) + + // Stage one image for the next send: the budget and the type are + // checked here — loudly, never a silent drop — the bytes copied into + // the cache's staged dir, and a downscaled chip preview decoded off + // the main thread. + fun stageImage(uri: Uri, name: String, mediaType: String?) { + stageError = null + if (staged.size >= MAX_STAGED_IMAGES) { + stageError = ImageStageError.TooMany(name, MAX_STAGED_IMAGES) + return + } + val type = mediaType?.takeIf { it in IMAGE_MEDIA_TYPES } + ?: MIME_BY_EXTENSION[name.substringAfterLast('.', "").lowercase()] + if (type == null) { + stageError = ImageStageError.BadType(name) + return + } + scope.launch { + when (val outcome = stageImageTask(context, uri, name, type)) { + is StageOutcome.Staged -> staged = staged + outcome.chip + is StageOutcome.Failed -> stageError = outcome.error + } + } + } + + val galleryLauncher = rememberLauncherForActivityResult( + ActivityResultContracts.PickVisualMedia(), + ) { uri -> + if (uri != null) { + val type = context.contentResolver.getType(uri) + val name = queryDisplayName(context, uri) ?: "photo." + typeFallbackExtension(type) + stageImage(uri, name, type) + } + } + + val cameraLauncher = rememberLauncherForActivityResult( + ActivityResultContracts.TakePicture(), + ) { shot -> + val target = cameraTarget + if (shot && target != null) { + stageImage(target.toUri(), cameraShotName, "image/jpeg") + } + } + // The claw's presentation identity: the avatar picture (when one is // set), the initials glyph otherwise, and the agent's name on every // turn — the same face the web UI shows. @@ -124,6 +200,27 @@ fun ChatScreen(container: AppContainer, sessionKey: String, onBack: () -> Unit) ) } + // The staged chips sit above the input row like the web composer's + // paste-chip row: thumbnail, ▣N name size, and a drop affordance. + if (staged.isNotEmpty()) { + Row( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 12.dp) + .horizontalScroll(rememberScrollState()), + horizontalArrangement = Arrangement.spacedBy(8.dp), + verticalAlignment = Alignment.Bottom, + ) { + staged.forEachIndexed { index, chip -> + StagedChipView(chip, index + 1) { staged = staged - chip } + } + } + } + + stageError?.let { error -> + ImageErrorRow(error) { stageError = null } + } + Row( modifier = Modifier .fillMaxWidth() @@ -131,6 +228,16 @@ fun ChatScreen(container: AppContainer, sessionKey: String, onBack: () -> Unit) verticalAlignment = Alignment.Bottom, horizontalArrangement = Arrangement.spacedBy(8.dp), ) { + IconButton( + onClick = { attachOpen = true }, + enabled = !controller.sending, + ) { + Icon( + Icons.Filled.Add, + contentDescription = stringResource(R.string.chat_attach), + tint = JadeiteText3, + ) + } OutlinedTextField( value = draft, onValueChange = { draft = it }, @@ -152,12 +259,59 @@ fun ChatScreen(container: AppContainer, sessionKey: String, onBack: () -> Unit) enabled = !controller.sending && draft.isNotBlank(), ) { val text = draft + val sending = staged draft = "" + staged = emptyList() + stageError = null scope.launch { - if (!controller.send(text)) draft = text + if (!controller.send(text, sending.map { it.image })) { + draft = text + staged = sending + } } } } + + // The attach door: gallery pick or camera shot, the only two ways + // images enter the composer. + if (attachOpen) { + AlertDialog( + onDismissRequest = { attachOpen = false }, + title = { Text(stringResource(R.string.chat_attach)) }, + text = { + Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { + OutlinedButton( + onClick = { + attachOpen = false + galleryLauncher.launch( + PickVisualMediaRequest( + ActivityResultContracts.PickVisualMedia.ImageOnly, + ), + ) + }, + modifier = Modifier.weight(1f), + ) { + Text(stringResource(R.string.chat_attach_gallery)) + } + OutlinedButton( + onClick = { + attachOpen = false + newCameraTarget(context)?.let { target -> + cameraTarget = target.toString() + cameraLauncher.launch(target) + } ?: run { + stageError = ImageStageError.ReadFailed(cameraLabel) + } + }, + modifier = Modifier.weight(1f), + ) { + Text(stringResource(R.string.chat_attach_camera)) + } + } + }, + confirmButton = {}, + ) + } } } @@ -247,7 +401,11 @@ private fun SendButton(enabled: Boolean, onClick: () -> Unit) { private sealed interface TurnGroup { val key: String - data class User(val id: Long, val text: String) : TurnGroup { + data class User( + val id: Long, + val text: String, + val images: List, + ) : TurnGroup { override val key: String = "u$id" } @@ -260,7 +418,7 @@ private fun turnGroups(items: List): List { val groups = mutableListOf() for (item in items) { when (item) { - is ChatController.UserMessage -> groups += TurnGroup.User(item.id, item.text) + is ChatController.UserMessage -> groups += TurnGroup.User(item.id, item.text, item.images) else -> { val last = groups.lastOrNull() if (last is TurnGroup.Agent) last.items += item else groups += TurnGroup.Agent(mutableListOf(item)) @@ -297,6 +455,19 @@ private fun TurnGroupRow( .weight(1f, fill = false), verticalArrangement = Arrangement.spacedBy(4.dp), ) { + // The message's images sit above its text, the web + // sent row's layout: a locally sent row previews its + // own staged files, a replayed one fetches by id. + if (group.images.isNotEmpty()) { + Row( + horizontalArrangement = Arrangement.spacedBy(6.dp), + modifier = Modifier.horizontalScroll(rememberScrollState()), + ) { + group.images.forEach { image -> + SentImageThumb(image, container) + } + } + } // Embedded pastes render as chips, not payloads; the // text around them stays verbatim. var chip = 0 @@ -577,6 +748,267 @@ private fun lineCount(text: String): Int { * payload's line count. */ private fun chipLabel(paste: String, chip: Int): String = "▣$chip ${lineCount(paste)}L" +// --- staged images ---------------------------------------------------- + +// One staged image as the composer holds it: the sendable part (file, +// wire type, decoded size) plus what only the UI needs — the label and +// the downscaled preview. +private data class StagedChip( + val image: ChatController.StagedImage, + val name: String, + val thumb: ImageBitmap?, +) + +// Why an image refused to stage — failure data, resolved to text at +// render (the UiMessage rule), never a silent drop. +private sealed interface ImageStageError { + data class TooMany(val name: String, val max: Int) : ImageStageError + data class TooLarge(val name: String) : ImageStageError + data class BadType(val name: String) : ImageStageError + data class ReadFailed(val name: String) : ImageStageError +} + +private sealed interface StageOutcome { + data class Staged(val chip: StagedChip) : StageOutcome + data class Failed(val error: ImageStageError) : StageOutcome +} + +// The claw accepts png, jpeg, gif, and webp only; anything else the +// picker offers (heic, raw camera types) refuses loudly at stage time. +private val IMAGE_MEDIA_TYPES = listOf("image/png", "image/jpeg", "image/gif", "image/webp") + +private val MIME_BY_EXTENSION = mapOf( + "png" to "image/png", + "jpg" to "image/jpeg", + "jpeg" to "image/jpeg", + "gif" to "image/gif", + "webp" to "image/webp", +) + +// Chip previews at staging, transcript previews at render — both far +// under the decoded sizes they stand for. +private const val STAGE_THUMB_PX = 256 +private const val TRANSCRIPT_THUMB_PX = 512 + +/** Copies a picked or shot image into the cache's staged dir, size-checked + * against the claw's budget, with a downscaled chip preview decoded off + * the main thread. Every refusal answers a failure the composer shows. */ +private suspend fun stageImageTask( + context: Context, + uri: Uri, + name: String, + mediaType: String, +): StageOutcome = withContext(Dispatchers.IO) { + // The read can fail on a dead grant (SecurityException) or a gone + // source (IOException); both stage as unreadable, both visible. + val bytes = try { + context.contentResolver.openInputStream(uri)?.use { stream -> stream.readBytes() } + } catch (e: IOException) { + null + } catch (e: SecurityException) { + null + } + when { + bytes == null -> StageOutcome.Failed(ImageStageError.ReadFailed(name)) + bytes.size > MAX_IMAGE_BYTES -> StageOutcome.Failed(ImageStageError.TooLarge(name)) + else -> { + val dir = File(context.cacheDir, "staged") + when { + !dir.exists() && !dir.mkdirs() -> + StageOutcome.Failed(ImageStageError.ReadFailed(name)) + else -> try { + // Synthetic on-disk name, never shown: the chip and + // the sent row label carry the pick's display name. + val file = File( + dir, + "staged-${System.currentTimeMillis()}.${stagedExtension(mediaType)}", + ) + file.writeBytes(bytes) + StageOutcome.Staged( + StagedChip( + image = ChatController.StagedImage(file, mediaType, bytes.size.toLong()), + name = name, + thumb = decodeThumbnail(bytes, STAGE_THUMB_PX), + ), + ) + } catch (e: IOException) { + StageOutcome.Failed(ImageStageError.ReadFailed(name)) + } + } + } + } +} + +/** The staged file's extension, from its accepted media type. */ +private fun stagedExtension(mediaType: String): String = + if (mediaType == "image/jpeg") "jpg" else mediaType.removePrefix("image/") + +/** A label's extension when the provider named no type: the image + * subtype when usable, png otherwise. */ +private fun typeFallbackExtension(mediaType: String?): String = + when (mediaType?.removePrefix("image/")) { + "gif" -> "gif" + "webp" -> "webp" + "jpeg" -> "jpg" + else -> "png" + } + +/** The pick's display name from the media provider; null when it names + * nothing — the caller falls back to a type-derived one. */ +private fun queryDisplayName(context: Context, uri: Uri): String? = runCatching { + context.contentResolver.query(uri, arrayOf(OpenableColumns.DISPLAY_NAME), null, null, null) + ?.use { cursor -> if (cursor.moveToFirst()) cursor.getString(0) else null } +}.getOrNull() + +/** A fresh cache file for a camera shot, mapped through the manifest's + * FileProvider: TakePicture writes straight into the content:// uri, and + * it needs no CAMERA permission. Null when the cache directory cannot + * be made — the composer surfaces that, it never drops silently. */ +private fun newCameraTarget(context: Context): Uri? { + val dir = File(context.cacheDir, "camera") + if (!dir.exists() && !dir.mkdirs()) return null + val file = File(dir, "shot-${System.currentTimeMillis()}.jpg") + return FileProvider.getUriForFile(context, context.packageName + ".fileprovider", file) +} + +/** A downscaled decode: inSampleSize halves until the longer side is + * near [maxDim], so a 12 MP camera shot never becomes a full bitmap in + * memory. Null when the bytes decode to nothing. */ +private fun decodeThumbnail(bytes: ByteArray, maxDim: Int): ImageBitmap? { + val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } + BitmapFactory.decodeByteArray(bytes, 0, bytes.size, bounds) + if (bounds.outWidth <= 0 || bounds.outHeight <= 0) return null + var sample = 1 + while (bounds.outWidth / (sample * 2) >= maxDim || bounds.outHeight / (sample * 2) >= maxDim) { + sample *= 2 + } + val options = BitmapFactory.Options().apply { inSampleSize = sample } + return BitmapFactory.decodeByteArray(bytes, 0, bytes.size, options)?.asImageBitmap() +} + +/** B/KB/MB with one decimal past 1 KB — the chip label's size, the web + * composer's format. */ +private fun formatSize(size: Long): String = when { + size < 1024 -> "$size B" + size < 1024 * 1024 -> String.format(Locale.US, "%.1f KB", size / 1024.0) + else -> String.format(Locale.US, "%.1f MB", size / (1024.0 * 1024.0)) +} + +// A staged image's chip, the paste chip's sibling: downscaled thumbnail, +// the ▣N name size label, and a drop affordance. +@Composable +private fun StagedChipView(chip: StagedChip, number: Int, onRemove: () -> Unit) { + Surface( + shape = MaterialTheme.shapes.extraSmall, + color = JadeiteSunken, + border = BorderStroke(1.dp, JadeiteBorder), + ) { + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(6.dp), + modifier = Modifier.padding(start = 6.dp, end = 6.dp, top = 4.dp, bottom = 4.dp), + ) { + chip.thumb?.let { thumb -> + Image( + bitmap = thumb, + contentDescription = chip.name, + contentScale = ContentScale.Crop, + modifier = Modifier + .size(28.dp) + .clip(MaterialTheme.shapes.extraSmall), + ) + } + Text( + "▣$number ${chip.name} ${formatSize(chip.image.sizeBytes)}", + style = MaterialTheme.typography.labelSmall, + fontFamily = JadeiteMono, + color = JadeiteText3, + ) + Icon( + Icons.Filled.Close, + contentDescription = stringResource(R.string.chat_image_drop, chip.name), + tint = JadeiteText3, + modifier = Modifier + .size(14.dp) + .clickable(onClick = onRemove), + ) + } + } +} + +// A refused image's inline error with its dismiss — the web composer's +// role=alert row. +@Composable +private fun ImageErrorRow(error: ImageStageError, onDismiss: () -> Unit) { + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(6.dp), + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 16.dp, vertical = 2.dp), + ) { + Text( + when (error) { + is ImageStageError.TooMany -> + stringResource(R.string.chat_image_too_many, error.name, error.max) + is ImageStageError.TooLarge -> + stringResource(R.string.chat_image_too_large, error.name) + is ImageStageError.BadType -> + stringResource(R.string.chat_image_bad_type, error.name) + is ImageStageError.ReadFailed -> + stringResource(R.string.chat_image_read_failed, error.name) + }, + color = JadeiteError, + style = MaterialTheme.typography.bodySmall, + modifier = Modifier.weight(1f), + ) + Icon( + Icons.Filled.Close, + contentDescription = stringResource(R.string.chat_image_dismiss), + tint = JadeiteText3, + modifier = Modifier + .size(16.dp) + .clickable(onClick = onDismiss), + ) + } +} + +// One image a sent row carries: the local file decodes in place; a +// replayed row fetches its bytes bearer-guarded once per row lifetime — +// the attachment card's own caching pattern. The preview is a downscaled +// decode, capped at the attachment card's 240 dp. +@Composable +private fun SentImageThumb(image: ChatController.SentImage, container: AppContainer) { + var thumb by remember(image.attachmentId, image.file) { mutableStateOf(null) } + LaunchedEffect(image.attachmentId, image.file) { + thumb = withContext(Dispatchers.IO) { + val bytes = when { + image.file != null -> runCatching { image.file.readBytes() }.getOrNull() + image.attachmentId != null -> when ( + val result = container.api.attachment(image.attachmentId) + ) { + is ClawResult.Ok -> result.value + else -> null + } + else -> null + } + bytes?.let { decodeThumbnail(it, TRANSCRIPT_THUMB_PX) } + } + } + thumb?.let { bitmap -> + Image( + bitmap = bitmap, + contentDescription = stringResource(R.string.chat_sent_image), + contentScale = ContentScale.Fit, + modifier = Modifier + .widthIn(max = 240.dp) + .heightIn(max = 240.dp) + .clip(MaterialTheme.shapes.small) + .border(1.dp, JadeiteBorder, MaterialTheme.shapes.small), + ) + } +} + // --- attachments ------------------------------------------------------ // Only small images preview inline: there is no image loader in the diff --git a/android/app/src/main/java/rs/rhema/phone/ui/SettingsScreen.kt b/android/app/src/main/java/rs/rhema/phone/ui/SettingsScreen.kt index cd6ee42..d43c657 100644 --- a/android/app/src/main/java/rs/rhema/phone/ui/SettingsScreen.kt +++ b/android/app/src/main/java/rs/rhema/phone/ui/SettingsScreen.kt @@ -434,7 +434,7 @@ private fun ProvidersCard(container: AppContainer) { options = when (val l = listing) { is ListingState.Available -> listOf(DEFAULT_MODEL to stringResource(R.string.providers_default_option)) + - l.models.map { it.id to it.id } + l.models.map { it.id to modelOptionLabel(it) } else -> listOf(DEFAULT_MODEL to stringResource(R.string.providers_default_option)) }, enabled = !saving, @@ -532,6 +532,18 @@ private fun ProvidersCard(container: AppContainer) { } } +/** A model option's label: the id plus the listing's vision verdict, + * spelled the way the web UI and the TUI's picker spell it — a mark + * when the model takes images, a bare "no vision" when it does not, + * and nothing when the key is absent: unknown is no verdict, never a + * false. */ +@Composable +private fun modelOptionLabel(model: ModelCard): String = model.id + when (model.supportsVision) { + true -> stringResource(R.string.vision_badge) + false -> stringResource(R.string.vision_badge_absent) + null -> "" +} + private sealed interface ListingState { data object Idle : ListingState data object Loading : ListingState diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml index e4af900..300754d 100644 --- a/android/app/src/main/res/values/strings.xml +++ b/android/app/src/main/res/values/strings.xml @@ -39,6 +39,17 @@ Open No app on this phone opens this file type. Download failed: %1$s + Attach images + Gallery + Camera + camera shot.jpg + Drop %1$s + Dismiss + %1$s: at most %2$d images a message + %1$s: over the 5 MiB decoded budget + %1$s: not a png, jpeg, gif, or webp image + %1$s: could not be read + Attached image The secret the agent asked for Send secret Approve @@ -148,5 +159,10 @@ Other keys rhema + + + " ◉ vision" + " no vision" diff --git a/android/app/src/main/res/xml/file_paths.xml b/android/app/src/main/res/xml/file_paths.xml index 8043193..89de310 100644 --- a/android/app/src/main/res/xml/file_paths.xml +++ b/android/app/src/main/res/xml/file_paths.xml @@ -3,4 +3,7 @@ hand-off; FileProvider maps files under it to content:// uris. --> + +