feat(android): attach photos and camera shots
The (+) door stages gallery picks and camera shots as removable
chips against the daemon's own budget (eight a message, 5 MiB
decoded each, refusals surfaced, never silent drops); send reads
the files back as bare base64, the old {"text"} body unchanged
when there are none. Sent rows preview their staged files at once
and a replay fetches by attachment id like any attachment card.
Model options carry the vision badge only when the listing knows.
This commit is contained in:
1 parent
6efb69331d
commit
43d4c86d5e
8 files changed
+605
-22
No files matched your search
@@ -1,5 +1,6 @@
|
||||
package rs.rhema.phone.chat
|
||||
|
||||
import android.util.Base64
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.setValue
|
||||
@@ -9,14 +10,25 @@ import rs.rhema.phone.data.SessionCache
|
||||
import rs.rhema.phone.net.ClawApi
|
||||
import rs.rhema.phone.net.ClawEvent
|
||||
import rs.rhema.phone.net.ClawResult
|
||||
import rs.rhema.phone.net.OutboundImage
|
||||
import rs.rhema.phone.net.SseFrame
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.Job
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.coroutines.NonCancellable
|
||||
import java.io.File
|
||||
import java.io.IOException
|
||||
|
||||
// The claw's own image budget (POST .../messages): eight images a
|
||||
// message, 5 MiB decoded each. The composer enforces it at stage time;
|
||||
// send() checks again so a stale staged file cannot smuggle an
|
||||
// oversized body past it.
|
||||
const val MAX_STAGED_IMAGES = 8
|
||||
const val MAX_IMAGE_BYTES = 5L * 1024 * 1024
|
||||
|
||||
// Holds the live transcript of one open chat screen. The transcript is
|
||||
// seeded from the session cache before the first frame and kept in the
|
||||
@@ -34,7 +46,20 @@ class ChatController(
|
||||
val id: Long
|
||||
}
|
||||
|
||||
data class UserMessage(override val id: Long, val text: String) : Item
|
||||
// One image a user row renders: the staged file behind a message
|
||||
// this surface just sent (previews locally, no id yet), or the
|
||||
// attachment id a replayed row names (bytes live at the claw).
|
||||
data class SentImage(
|
||||
val attachmentId: String?,
|
||||
val file: File?,
|
||||
val mediaType: String,
|
||||
)
|
||||
|
||||
data class UserMessage(
|
||||
override val id: Long,
|
||||
val text: String,
|
||||
val images: List<SentImage> = emptyList(),
|
||||
) : Item
|
||||
data class Thinking(override val id: Long, val text: String) : Item
|
||||
data class ToolCall(
|
||||
override val id: Long,
|
||||
@@ -158,22 +183,57 @@ class ChatController(
|
||||
streamJob = null
|
||||
}
|
||||
|
||||
// One image staged for the next send: the file the optimistic row
|
||||
// previews, the wire type the POST body carries, and its decoded
|
||||
// size for the budget's last look.
|
||||
data class StagedImage(val file: File, val mediaType: String, val sizeBytes: Long)
|
||||
|
||||
// The stream is already collecting by the time the user can press send,
|
||||
// so this preserves the claw's subscribe-then-post order. A failed POST
|
||||
// restores the draft in the UI. The turn's user_message event is the
|
||||
// so this preserves the claw's subscribe-then-post order. Images are
|
||||
// read and encoded here — the send path's own budget look (the composer
|
||||
// already enforced it at stage time); a violation or an unreadable file
|
||||
// fails the send with the error surfaced, never a silent drop. A failed
|
||||
// POST restores the draft in the UI. The turn's user_message event is the
|
||||
// transcript's own record of the question — replayed on a reload, and
|
||||
// echoed live right about when this returns — so the local row is only
|
||||
// appended when the echo has not already landed.
|
||||
suspend fun send(text: String): Boolean {
|
||||
suspend fun send(text: String, staged: List<StagedImage> = emptyList()): Boolean {
|
||||
if (staged.size > MAX_STAGED_IMAGES) {
|
||||
sendError = "over the $MAX_STAGED_IMAGES-image budget"
|
||||
return false
|
||||
}
|
||||
sending = true
|
||||
sendError = null
|
||||
val result = api.postMessage(sessionKey, text)
|
||||
val outbound = mutableListOf<OutboundImage>()
|
||||
for (image in staged) {
|
||||
try {
|
||||
val bytes = withContext(Dispatchers.IO) { image.file.readBytes() }
|
||||
if (bytes.size > MAX_IMAGE_BYTES) {
|
||||
sending = false
|
||||
sendError = "image over the 5 MiB decoded budget"
|
||||
return false
|
||||
}
|
||||
outbound += OutboundImage(
|
||||
image.mediaType,
|
||||
Base64.encodeToString(bytes, Base64.NO_WRAP),
|
||||
)
|
||||
} catch (e: IOException) {
|
||||
sending = false
|
||||
sendError = "image could not be read"
|
||||
return false
|
||||
}
|
||||
}
|
||||
val result = api.postMessage(sessionKey, text, outbound)
|
||||
sending = false
|
||||
return when (result) {
|
||||
is ClawResult.Ok -> {
|
||||
val last = items.lastOrNull()
|
||||
if (!(last is UserMessage && last.text == text)) {
|
||||
items = items + UserMessage(newId(), text)
|
||||
if (!(last is UserMessage && last.text == text && last.images.size == staged.size)) {
|
||||
items = items + UserMessage(
|
||||
newId(),
|
||||
text,
|
||||
staged.map { SentImage(attachmentId = null, file = it.file, mediaType = it.mediaType) },
|
||||
)
|
||||
}
|
||||
true
|
||||
}
|
||||
@@ -200,10 +260,15 @@ class ChatController(
|
||||
is ClawEvent.UserMessage -> {
|
||||
// The recorded question — replayed on reload, echoed live
|
||||
// for a message this surface just sent. An identical last
|
||||
// row is that echo's stand-in; skip it.
|
||||
// row — same text, same image count — is that echo's
|
||||
// stand-in; skip it, previews and all.
|
||||
val last = list.lastOrNull()
|
||||
if (!(last is UserMessage && last.text == event.text)) {
|
||||
list = list + UserMessage(newId(), event.text)
|
||||
if (!(last is UserMessage && last.text == event.text && last.images.size == event.images.size)) {
|
||||
list = list + UserMessage(
|
||||
newId(),
|
||||
event.text,
|
||||
event.images.map { SentImage(attachmentId = it.id, file = null, mediaType = it.mediaType) },
|
||||
)
|
||||
}
|
||||
}
|
||||
is ClawEvent.ThinkingDelta -> {
|
||||
|
||||
@@ -102,10 +102,16 @@ class ClawApi(
|
||||
|
||||
// The claw's documented dance: subscribe to the event stream first, then
|
||||
// POST; the reply arrives on the stream, not in the POST response (202).
|
||||
suspend fun postMessage(sessionKey: String, text: String): ClawResult<Unit> {
|
||||
// Staged images ride the body only when there are some (the request
|
||||
// type's default keeps the old shape byte-stable otherwise).
|
||||
suspend fun postMessage(
|
||||
sessionKey: String,
|
||||
text: String,
|
||||
images: List<OutboundImage> = emptyList(),
|
||||
): ClawResult<Unit> {
|
||||
val builder = v1("sessions", sessionKey, "messages")
|
||||
?: return ClawResult.Network("no base URL configured")
|
||||
val body = json.encodeToString(MessageRequest(text)).toRequestBody(jsonBody)
|
||||
val body = json.encodeToString(MessageRequest(text, images)).toRequestBody(jsonBody)
|
||||
return execute(builder.post(body).build()) { }
|
||||
}
|
||||
|
||||
|
||||
@@ -20,9 +20,20 @@ data class SessionInfo(
|
||||
@SerialName("last_outcome") val lastOutcome: JsonElement? = null,
|
||||
)
|
||||
|
||||
// POST /v1/sessions/{key}/messages
|
||||
// POST /v1/sessions/{key}/messages. The Json instance skips defaults,
|
||||
// so an empty image list keeps the old {"text"} body byte-for-byte.
|
||||
// Text stays required: the claw 422s an empty text even with images.
|
||||
@Serializable
|
||||
data class MessageRequest(val text: String)
|
||||
data class MessageRequest(val text: String, val images: List<OutboundImage> = emptyList())
|
||||
|
||||
// One image in the POST body: the full MIME type plus bare base64. The
|
||||
// claw's budget — at most eight, 5 MiB decoded each, png/jpeg/gif/webp
|
||||
// only — is enforced client-side at stage time (the composer).
|
||||
@Serializable
|
||||
data class OutboundImage(
|
||||
@SerialName("media_type") val mediaType: String,
|
||||
val data: String,
|
||||
)
|
||||
|
||||
// GET /v1/providers — the backend instances and the active pair. Each
|
||||
// row is one instance, named by its id: the kind unless a login named
|
||||
@@ -46,6 +57,9 @@ data class ModelCard(
|
||||
val id: String,
|
||||
@SerialName("display_name") val displayName: String? = null,
|
||||
val description: String? = null,
|
||||
// Whether the model takes image input, when the adapter knows:
|
||||
// absent means unknown — no verdict, never a false.
|
||||
@SerialName("supports_vision") val supportsVision: Boolean? = null,
|
||||
)
|
||||
|
||||
@Serializable
|
||||
@@ -97,6 +111,12 @@ data class RhemaNotification(
|
||||
@SerialName("created_at") val createdAt: Long,
|
||||
)
|
||||
|
||||
// One image a user_message names: the attachment id the bytes live
|
||||
// under (GET /v1/attachments/{id}, bearer-guarded) plus the MIME type —
|
||||
// megabytes never ride the stream.
|
||||
@Serializable
|
||||
data class UserImage(val id: String, @SerialName("media_type") val mediaType: String)
|
||||
|
||||
// Frames of GET /v1/sessions/{key}/events; "type" is the discriminator.
|
||||
// The stream opens with the session's recorded turns replayed before the
|
||||
// live feed, so a screen (re)builds its transcript from the replay.
|
||||
@@ -106,7 +126,12 @@ data class RhemaNotification(
|
||||
sealed interface ClawEvent {
|
||||
@Serializable
|
||||
@SerialName("user_message")
|
||||
data class UserMessage(val text: String = "") : ClawEvent
|
||||
// Rows recorded before images rode the stream carry no field at
|
||||
// all; absent folds to none, never a guessed shape.
|
||||
data class UserMessage(
|
||||
val text: String = "",
|
||||
val images: List<UserImage> = emptyList(),
|
||||
) : ClawEvent
|
||||
|
||||
@Serializable
|
||||
@SerialName("thinking_delta")
|
||||
|
||||
@@ -6,9 +6,16 @@ import android.content.Intent
|
||||
import android.graphics.BitmapFactory
|
||||
import android.net.Uri
|
||||
import androidx.activity.compose.BackHandler
|
||||
import androidx.activity.compose.rememberLauncherForActivityResult
|
||||
import androidx.activity.result.PickVisualMediaRequest
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.compose.foundation.BorderStroke
|
||||
import androidx.compose.foundation.Image
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.border
|
||||
import androidx.compose.foundation.clickable
|
||||
import androidx.compose.foundation.horizontalScroll
|
||||
import androidx.compose.foundation.rememberScrollState
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
@@ -21,12 +28,16 @@ import androidx.compose.foundation.layout.PaddingValues
|
||||
import androidx.compose.foundation.layout.IntrinsicSize
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.layout.widthIn
|
||||
import androidx.compose.foundation.lazy.LazyColumn
|
||||
import androidx.compose.foundation.lazy.items
|
||||
import androidx.compose.foundation.shape.CircleShape
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.automirrored.filled.ArrowBack
|
||||
import androidx.compose.material.icons.automirrored.filled.Send
|
||||
import androidx.compose.material.icons.filled.Add
|
||||
import androidx.compose.material.icons.filled.Close
|
||||
import androidx.compose.material3.AlertDialog
|
||||
import androidx.compose.material3.Button
|
||||
import androidx.compose.material3.HorizontalDivider
|
||||
import androidx.compose.material3.Icon
|
||||
@@ -49,6 +60,7 @@ import androidx.compose.runtime.saveable.rememberSaveable
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.draw.clip
|
||||
import androidx.compose.ui.graphics.ImageBitmap
|
||||
import androidx.compose.ui.graphics.asImageBitmap
|
||||
import androidx.compose.ui.layout.ContentScale
|
||||
@@ -60,8 +72,11 @@ import androidx.compose.ui.text.input.PasswordVisualTransformation
|
||||
import androidx.compose.ui.text.style.TextOverflow
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.core.content.FileProvider
|
||||
import androidx.core.net.toUri
|
||||
import rs.rhema.phone.R
|
||||
import rs.rhema.phone.chat.ChatController
|
||||
import rs.rhema.phone.chat.MAX_IMAGE_BYTES
|
||||
import rs.rhema.phone.chat.MAX_STAGED_IMAGES
|
||||
import rs.rhema.phone.data.AgentIdentityCache
|
||||
import rs.rhema.phone.di.AppContainer
|
||||
import rs.rhema.phone.net.ClawResult
|
||||
@@ -70,6 +85,8 @@ import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.io.File
|
||||
import java.io.IOException
|
||||
import java.util.Locale
|
||||
import android.provider.OpenableColumns
|
||||
|
||||
// The transcript screen, the design's chat panel as a phone screen:
|
||||
// avatar rows per turn (the tool cards with their one-sided accent
|
||||
@@ -89,6 +106,65 @@ fun ChatScreen(container: AppContainer, sessionKey: String, onBack: () -> Unit)
|
||||
|
||||
var draft by rememberSaveable { mutableStateOf("") }
|
||||
|
||||
// Images staged for the next send, held as files in the app cache:
|
||||
// the picker's read grant dies with the activity result, while the
|
||||
// send reads the staged file whenever the message leaves.
|
||||
var staged by remember { mutableStateOf<List<StagedChip>>(emptyList()) }
|
||||
var stageError by remember { mutableStateOf<ImageStageError?>(null) }
|
||||
var attachOpen by remember { mutableStateOf(false) }
|
||||
// The camera's content:// target, remembered so the result callback
|
||||
// still knows what was written even after a configuration change.
|
||||
var cameraTarget by rememberSaveable { mutableStateOf<String?>(null) }
|
||||
val context = LocalContext.current
|
||||
// Labels resolve in composition, not at callback time: a lookup
|
||||
// through LocalContext.current freezes the first composition's
|
||||
// locale (the same rule the attachment card's templates follow).
|
||||
val cameraLabel = stringResource(R.string.chat_attach_camera)
|
||||
val cameraShotName = stringResource(R.string.chat_camera_shot_name)
|
||||
|
||||
// Stage one image for the next send: the budget and the type are
|
||||
// checked here — loudly, never a silent drop — the bytes copied into
|
||||
// the cache's staged dir, and a downscaled chip preview decoded off
|
||||
// the main thread.
|
||||
fun stageImage(uri: Uri, name: String, mediaType: String?) {
|
||||
stageError = null
|
||||
if (staged.size >= MAX_STAGED_IMAGES) {
|
||||
stageError = ImageStageError.TooMany(name, MAX_STAGED_IMAGES)
|
||||
return
|
||||
}
|
||||
val type = mediaType?.takeIf { it in IMAGE_MEDIA_TYPES }
|
||||
?: MIME_BY_EXTENSION[name.substringAfterLast('.', "").lowercase()]
|
||||
if (type == null) {
|
||||
stageError = ImageStageError.BadType(name)
|
||||
return
|
||||
}
|
||||
scope.launch {
|
||||
when (val outcome = stageImageTask(context, uri, name, type)) {
|
||||
is StageOutcome.Staged -> staged = staged + outcome.chip
|
||||
is StageOutcome.Failed -> stageError = outcome.error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val galleryLauncher = rememberLauncherForActivityResult(
|
||||
ActivityResultContracts.PickVisualMedia(),
|
||||
) { uri ->
|
||||
if (uri != null) {
|
||||
val type = context.contentResolver.getType(uri)
|
||||
val name = queryDisplayName(context, uri) ?: "photo." + typeFallbackExtension(type)
|
||||
stageImage(uri, name, type)
|
||||
}
|
||||
}
|
||||
|
||||
val cameraLauncher = rememberLauncherForActivityResult(
|
||||
ActivityResultContracts.TakePicture(),
|
||||
) { shot ->
|
||||
val target = cameraTarget
|
||||
if (shot && target != null) {
|
||||
stageImage(target.toUri(), cameraShotName, "image/jpeg")
|
||||
}
|
||||
}
|
||||
|
||||
// The claw's presentation identity: the avatar picture (when one is
|
||||
// set), the initials glyph otherwise, and the agent's name on every
|
||||
// turn — the same face the web UI shows.
|
||||
@@ -124,6 +200,27 @@ fun ChatScreen(container: AppContainer, sessionKey: String, onBack: () -> Unit)
|
||||
)
|
||||
}
|
||||
|
||||
// The staged chips sit above the input row like the web composer's
|
||||
// paste-chip row: thumbnail, ▣N name size, and a drop affordance.
|
||||
if (staged.isNotEmpty()) {
|
||||
Row(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.padding(horizontal = 12.dp)
|
||||
.horizontalScroll(rememberScrollState()),
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
verticalAlignment = Alignment.Bottom,
|
||||
) {
|
||||
staged.forEachIndexed { index, chip ->
|
||||
StagedChipView(chip, index + 1) { staged = staged - chip }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stageError?.let { error ->
|
||||
ImageErrorRow(error) { stageError = null }
|
||||
}
|
||||
|
||||
Row(
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
@@ -131,6 +228,16 @@ fun ChatScreen(container: AppContainer, sessionKey: String, onBack: () -> Unit)
|
||||
verticalAlignment = Alignment.Bottom,
|
||||
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||
) {
|
||||
IconButton(
|
||||
onClick = { attachOpen = true },
|
||||
enabled = !controller.sending,
|
||||
) {
|
||||
Icon(
|
||||
Icons.Filled.Add,
|
||||
contentDescription = stringResource(R.string.chat_attach),
|
||||
tint = JadeiteText3,
|
||||
)
|
||||
}
|
||||
OutlinedTextField(
|
||||
value = draft,
|
||||
onValueChange = { draft = it },
|
||||
@@ -152,12 +259,59 @@ fun ChatScreen(container: AppContainer, sessionKey: String, onBack: () -> Unit)
|
||||
enabled = !controller.sending && draft.isNotBlank(),
|
||||
) {
|
||||
val text = draft
|
||||
val sending = staged
|
||||
draft = ""
|
||||
staged = emptyList()
|
||||
stageError = null
|
||||
scope.launch {
|
||||
if (!controller.send(text)) draft = text
|
||||
if (!controller.send(text, sending.map { it.image })) {
|
||||
draft = text
|
||||
staged = sending
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The attach door: gallery pick or camera shot, the only two ways
|
||||
// images enter the composer.
|
||||
if (attachOpen) {
|
||||
AlertDialog(
|
||||
onDismissRequest = { attachOpen = false },
|
||||
title = { Text(stringResource(R.string.chat_attach)) },
|
||||
text = {
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
OutlinedButton(
|
||||
onClick = {
|
||||
attachOpen = false
|
||||
galleryLauncher.launch(
|
||||
PickVisualMediaRequest(
|
||||
ActivityResultContracts.PickVisualMedia.ImageOnly,
|
||||
),
|
||||
)
|
||||
},
|
||||
modifier = Modifier.weight(1f),
|
||||
) {
|
||||
Text(stringResource(R.string.chat_attach_gallery))
|
||||
}
|
||||
OutlinedButton(
|
||||
onClick = {
|
||||
attachOpen = false
|
||||
newCameraTarget(context)?.let { target ->
|
||||
cameraTarget = target.toString()
|
||||
cameraLauncher.launch(target)
|
||||
} ?: run {
|
||||
stageError = ImageStageError.ReadFailed(cameraLabel)
|
||||
}
|
||||
},
|
||||
modifier = Modifier.weight(1f),
|
||||
) {
|
||||
Text(stringResource(R.string.chat_attach_camera))
|
||||
}
|
||||
}
|
||||
},
|
||||
confirmButton = {},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -247,7 +401,11 @@ private fun SendButton(enabled: Boolean, onClick: () -> Unit) {
|
||||
private sealed interface TurnGroup {
|
||||
val key: String
|
||||
|
||||
data class User(val id: Long, val text: String) : TurnGroup {
|
||||
data class User(
|
||||
val id: Long,
|
||||
val text: String,
|
||||
val images: List<ChatController.SentImage>,
|
||||
) : TurnGroup {
|
||||
override val key: String = "u$id"
|
||||
}
|
||||
|
||||
@@ -260,7 +418,7 @@ private fun turnGroups(items: List<ChatController.Item>): List<TurnGroup> {
|
||||
val groups = mutableListOf<TurnGroup>()
|
||||
for (item in items) {
|
||||
when (item) {
|
||||
is ChatController.UserMessage -> groups += TurnGroup.User(item.id, item.text)
|
||||
is ChatController.UserMessage -> groups += TurnGroup.User(item.id, item.text, item.images)
|
||||
else -> {
|
||||
val last = groups.lastOrNull()
|
||||
if (last is TurnGroup.Agent) last.items += item else groups += TurnGroup.Agent(mutableListOf(item))
|
||||
@@ -297,6 +455,19 @@ private fun TurnGroupRow(
|
||||
.weight(1f, fill = false),
|
||||
verticalArrangement = Arrangement.spacedBy(4.dp),
|
||||
) {
|
||||
// The message's images sit above its text, the web
|
||||
// sent row's layout: a locally sent row previews its
|
||||
// own staged files, a replayed one fetches by id.
|
||||
if (group.images.isNotEmpty()) {
|
||||
Row(
|
||||
horizontalArrangement = Arrangement.spacedBy(6.dp),
|
||||
modifier = Modifier.horizontalScroll(rememberScrollState()),
|
||||
) {
|
||||
group.images.forEach { image ->
|
||||
SentImageThumb(image, container)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Embedded pastes render as chips, not payloads; the
|
||||
// text around them stays verbatim.
|
||||
var chip = 0
|
||||
@@ -577,6 +748,267 @@ private fun lineCount(text: String): Int {
|
||||
* payload's line count. */
|
||||
private fun chipLabel(paste: String, chip: Int): String = "▣$chip ${lineCount(paste)}L"
|
||||
|
||||
// --- staged images ----------------------------------------------------
|
||||
|
||||
// One staged image as the composer holds it: the sendable part (file,
|
||||
// wire type, decoded size) plus what only the UI needs — the label and
|
||||
// the downscaled preview.
|
||||
private data class StagedChip(
|
||||
val image: ChatController.StagedImage,
|
||||
val name: String,
|
||||
val thumb: ImageBitmap?,
|
||||
)
|
||||
|
||||
// Why an image refused to stage — failure data, resolved to text at
|
||||
// render (the UiMessage rule), never a silent drop.
|
||||
private sealed interface ImageStageError {
|
||||
data class TooMany(val name: String, val max: Int) : ImageStageError
|
||||
data class TooLarge(val name: String) : ImageStageError
|
||||
data class BadType(val name: String) : ImageStageError
|
||||
data class ReadFailed(val name: String) : ImageStageError
|
||||
}
|
||||
|
||||
private sealed interface StageOutcome {
|
||||
data class Staged(val chip: StagedChip) : StageOutcome
|
||||
data class Failed(val error: ImageStageError) : StageOutcome
|
||||
}
|
||||
|
||||
// The claw accepts png, jpeg, gif, and webp only; anything else the
|
||||
// picker offers (heic, raw camera types) refuses loudly at stage time.
|
||||
private val IMAGE_MEDIA_TYPES = listOf("image/png", "image/jpeg", "image/gif", "image/webp")
|
||||
|
||||
private val MIME_BY_EXTENSION = mapOf(
|
||||
"png" to "image/png",
|
||||
"jpg" to "image/jpeg",
|
||||
"jpeg" to "image/jpeg",
|
||||
"gif" to "image/gif",
|
||||
"webp" to "image/webp",
|
||||
)
|
||||
|
||||
// Chip previews at staging, transcript previews at render — both far
|
||||
// under the decoded sizes they stand for.
|
||||
private const val STAGE_THUMB_PX = 256
|
||||
private const val TRANSCRIPT_THUMB_PX = 512
|
||||
|
||||
/** Copies a picked or shot image into the cache's staged dir, size-checked
|
||||
* against the claw's budget, with a downscaled chip preview decoded off
|
||||
* the main thread. Every refusal answers a failure the composer shows. */
|
||||
private suspend fun stageImageTask(
|
||||
context: Context,
|
||||
uri: Uri,
|
||||
name: String,
|
||||
mediaType: String,
|
||||
): StageOutcome = withContext(Dispatchers.IO) {
|
||||
// The read can fail on a dead grant (SecurityException) or a gone
|
||||
// source (IOException); both stage as unreadable, both visible.
|
||||
val bytes = try {
|
||||
context.contentResolver.openInputStream(uri)?.use { stream -> stream.readBytes() }
|
||||
} catch (e: IOException) {
|
||||
null
|
||||
} catch (e: SecurityException) {
|
||||
null
|
||||
}
|
||||
when {
|
||||
bytes == null -> StageOutcome.Failed(ImageStageError.ReadFailed(name))
|
||||
bytes.size > MAX_IMAGE_BYTES -> StageOutcome.Failed(ImageStageError.TooLarge(name))
|
||||
else -> {
|
||||
val dir = File(context.cacheDir, "staged")
|
||||
when {
|
||||
!dir.exists() && !dir.mkdirs() ->
|
||||
StageOutcome.Failed(ImageStageError.ReadFailed(name))
|
||||
else -> try {
|
||||
// Synthetic on-disk name, never shown: the chip and
|
||||
// the sent row label carry the pick's display name.
|
||||
val file = File(
|
||||
dir,
|
||||
"staged-${System.currentTimeMillis()}.${stagedExtension(mediaType)}",
|
||||
)
|
||||
file.writeBytes(bytes)
|
||||
StageOutcome.Staged(
|
||||
StagedChip(
|
||||
image = ChatController.StagedImage(file, mediaType, bytes.size.toLong()),
|
||||
name = name,
|
||||
thumb = decodeThumbnail(bytes, STAGE_THUMB_PX),
|
||||
),
|
||||
)
|
||||
} catch (e: IOException) {
|
||||
StageOutcome.Failed(ImageStageError.ReadFailed(name))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The staged file's extension, from its accepted media type. */
|
||||
private fun stagedExtension(mediaType: String): String =
|
||||
if (mediaType == "image/jpeg") "jpg" else mediaType.removePrefix("image/")
|
||||
|
||||
/** A label's extension when the provider named no type: the image
|
||||
* subtype when usable, png otherwise. */
|
||||
private fun typeFallbackExtension(mediaType: String?): String =
|
||||
when (mediaType?.removePrefix("image/")) {
|
||||
"gif" -> "gif"
|
||||
"webp" -> "webp"
|
||||
"jpeg" -> "jpg"
|
||||
else -> "png"
|
||||
}
|
||||
|
||||
/** The pick's display name from the media provider; null when it names
|
||||
* nothing — the caller falls back to a type-derived one. */
|
||||
private fun queryDisplayName(context: Context, uri: Uri): String? = runCatching {
|
||||
context.contentResolver.query(uri, arrayOf(OpenableColumns.DISPLAY_NAME), null, null, null)
|
||||
?.use { cursor -> if (cursor.moveToFirst()) cursor.getString(0) else null }
|
||||
}.getOrNull()
|
||||
|
||||
/** A fresh cache file for a camera shot, mapped through the manifest's
|
||||
* FileProvider: TakePicture writes straight into the content:// uri, and
|
||||
* it needs no CAMERA permission. Null when the cache directory cannot
|
||||
* be made — the composer surfaces that, it never drops silently. */
|
||||
private fun newCameraTarget(context: Context): Uri? {
|
||||
val dir = File(context.cacheDir, "camera")
|
||||
if (!dir.exists() && !dir.mkdirs()) return null
|
||||
val file = File(dir, "shot-${System.currentTimeMillis()}.jpg")
|
||||
return FileProvider.getUriForFile(context, context.packageName + ".fileprovider", file)
|
||||
}
|
||||
|
||||
/** A downscaled decode: inSampleSize halves until the longer side is
|
||||
* near [maxDim], so a 12 MP camera shot never becomes a full bitmap in
|
||||
* memory. Null when the bytes decode to nothing. */
|
||||
private fun decodeThumbnail(bytes: ByteArray, maxDim: Int): ImageBitmap? {
|
||||
val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true }
|
||||
BitmapFactory.decodeByteArray(bytes, 0, bytes.size, bounds)
|
||||
if (bounds.outWidth <= 0 || bounds.outHeight <= 0) return null
|
||||
var sample = 1
|
||||
while (bounds.outWidth / (sample * 2) >= maxDim || bounds.outHeight / (sample * 2) >= maxDim) {
|
||||
sample *= 2
|
||||
}
|
||||
val options = BitmapFactory.Options().apply { inSampleSize = sample }
|
||||
return BitmapFactory.decodeByteArray(bytes, 0, bytes.size, options)?.asImageBitmap()
|
||||
}
|
||||
|
||||
/** B/KB/MB with one decimal past 1 KB — the chip label's size, the web
|
||||
* composer's format. */
|
||||
private fun formatSize(size: Long): String = when {
|
||||
size < 1024 -> "$size B"
|
||||
size < 1024 * 1024 -> String.format(Locale.US, "%.1f KB", size / 1024.0)
|
||||
else -> String.format(Locale.US, "%.1f MB", size / (1024.0 * 1024.0))
|
||||
}
|
||||
|
||||
// A staged image's chip, the paste chip's sibling: downscaled thumbnail,
|
||||
// the ▣N name size label, and a drop affordance.
|
||||
@Composable
|
||||
private fun StagedChipView(chip: StagedChip, number: Int, onRemove: () -> Unit) {
|
||||
Surface(
|
||||
shape = MaterialTheme.shapes.extraSmall,
|
||||
color = JadeiteSunken,
|
||||
border = BorderStroke(1.dp, JadeiteBorder),
|
||||
) {
|
||||
Row(
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(6.dp),
|
||||
modifier = Modifier.padding(start = 6.dp, end = 6.dp, top = 4.dp, bottom = 4.dp),
|
||||
) {
|
||||
chip.thumb?.let { thumb ->
|
||||
Image(
|
||||
bitmap = thumb,
|
||||
contentDescription = chip.name,
|
||||
contentScale = ContentScale.Crop,
|
||||
modifier = Modifier
|
||||
.size(28.dp)
|
||||
.clip(MaterialTheme.shapes.extraSmall),
|
||||
)
|
||||
}
|
||||
Text(
|
||||
"▣$number ${chip.name} ${formatSize(chip.image.sizeBytes)}",
|
||||
style = MaterialTheme.typography.labelSmall,
|
||||
fontFamily = JadeiteMono,
|
||||
color = JadeiteText3,
|
||||
)
|
||||
Icon(
|
||||
Icons.Filled.Close,
|
||||
contentDescription = stringResource(R.string.chat_image_drop, chip.name),
|
||||
tint = JadeiteText3,
|
||||
modifier = Modifier
|
||||
.size(14.dp)
|
||||
.clickable(onClick = onRemove),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A refused image's inline error with its dismiss — the web composer's
|
||||
// role=alert row.
|
||||
@Composable
|
||||
private fun ImageErrorRow(error: ImageStageError, onDismiss: () -> Unit) {
|
||||
Row(
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
horizontalArrangement = Arrangement.spacedBy(6.dp),
|
||||
modifier = Modifier
|
||||
.fillMaxWidth()
|
||||
.padding(horizontal = 16.dp, vertical = 2.dp),
|
||||
) {
|
||||
Text(
|
||||
when (error) {
|
||||
is ImageStageError.TooMany ->
|
||||
stringResource(R.string.chat_image_too_many, error.name, error.max)
|
||||
is ImageStageError.TooLarge ->
|
||||
stringResource(R.string.chat_image_too_large, error.name)
|
||||
is ImageStageError.BadType ->
|
||||
stringResource(R.string.chat_image_bad_type, error.name)
|
||||
is ImageStageError.ReadFailed ->
|
||||
stringResource(R.string.chat_image_read_failed, error.name)
|
||||
},
|
||||
color = JadeiteError,
|
||||
style = MaterialTheme.typography.bodySmall,
|
||||
modifier = Modifier.weight(1f),
|
||||
)
|
||||
Icon(
|
||||
Icons.Filled.Close,
|
||||
contentDescription = stringResource(R.string.chat_image_dismiss),
|
||||
tint = JadeiteText3,
|
||||
modifier = Modifier
|
||||
.size(16.dp)
|
||||
.clickable(onClick = onDismiss),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// One image a sent row carries: the local file decodes in place; a
|
||||
// replayed row fetches its bytes bearer-guarded once per row lifetime —
|
||||
// the attachment card's own caching pattern. The preview is a downscaled
|
||||
// decode, capped at the attachment card's 240 dp.
|
||||
@Composable
|
||||
private fun SentImageThumb(image: ChatController.SentImage, container: AppContainer) {
|
||||
var thumb by remember(image.attachmentId, image.file) { mutableStateOf<ImageBitmap?>(null) }
|
||||
LaunchedEffect(image.attachmentId, image.file) {
|
||||
thumb = withContext(Dispatchers.IO) {
|
||||
val bytes = when {
|
||||
image.file != null -> runCatching { image.file.readBytes() }.getOrNull()
|
||||
image.attachmentId != null -> when (
|
||||
val result = container.api.attachment(image.attachmentId)
|
||||
) {
|
||||
is ClawResult.Ok -> result.value
|
||||
else -> null
|
||||
}
|
||||
else -> null
|
||||
}
|
||||
bytes?.let { decodeThumbnail(it, TRANSCRIPT_THUMB_PX) }
|
||||
}
|
||||
}
|
||||
thumb?.let { bitmap ->
|
||||
Image(
|
||||
bitmap = bitmap,
|
||||
contentDescription = stringResource(R.string.chat_sent_image),
|
||||
contentScale = ContentScale.Fit,
|
||||
modifier = Modifier
|
||||
.widthIn(max = 240.dp)
|
||||
.heightIn(max = 240.dp)
|
||||
.clip(MaterialTheme.shapes.small)
|
||||
.border(1.dp, JadeiteBorder, MaterialTheme.shapes.small),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// --- attachments ------------------------------------------------------
|
||||
|
||||
// Only small images preview inline: there is no image loader in the
|
||||
|
||||
@@ -434,7 +434,7 @@ private fun ProvidersCard(container: AppContainer) {
|
||||
options = when (val l = listing) {
|
||||
is ListingState.Available ->
|
||||
listOf(DEFAULT_MODEL to stringResource(R.string.providers_default_option)) +
|
||||
l.models.map { it.id to it.id }
|
||||
l.models.map { it.id to modelOptionLabel(it) }
|
||||
else -> listOf(DEFAULT_MODEL to stringResource(R.string.providers_default_option))
|
||||
},
|
||||
enabled = !saving,
|
||||
@@ -532,6 +532,18 @@ private fun ProvidersCard(container: AppContainer) {
|
||||
}
|
||||
}
|
||||
|
||||
/** A model option's label: the id plus the listing's vision verdict,
|
||||
* spelled the way the web UI and the TUI's picker spell it — a mark
|
||||
* when the model takes images, a bare "no vision" when it does not,
|
||||
* and nothing when the key is absent: unknown is no verdict, never a
|
||||
* false. */
|
||||
@Composable
|
||||
private fun modelOptionLabel(model: ModelCard): String = model.id + when (model.supportsVision) {
|
||||
true -> stringResource(R.string.vision_badge)
|
||||
false -> stringResource(R.string.vision_badge_absent)
|
||||
null -> ""
|
||||
}
|
||||
|
||||
private sealed interface ListingState {
|
||||
data object Idle : ListingState
|
||||
data object Loading : ListingState
|
||||
|
||||
@@ -39,6 +39,17 @@
|
||||
<string name="chat_attachment_open">Open</string>
|
||||
<string name="chat_attachment_open_failed">No app on this phone opens this file type.</string>
|
||||
<string name="chat_attachment_failed">Download failed: %1$s</string>
|
||||
<string name="chat_attach">Attach images</string>
|
||||
<string name="chat_attach_gallery">Gallery</string>
|
||||
<string name="chat_attach_camera">Camera</string>
|
||||
<string name="chat_camera_shot_name">camera shot.jpg</string>
|
||||
<string name="chat_image_drop">Drop %1$s</string>
|
||||
<string name="chat_image_dismiss">Dismiss</string>
|
||||
<string name="chat_image_too_many">%1$s: at most %2$d images a message</string>
|
||||
<string name="chat_image_too_large">%1$s: over the 5 MiB decoded budget</string>
|
||||
<string name="chat_image_bad_type">%1$s: not a png, jpeg, gif, or webp image</string>
|
||||
<string name="chat_image_read_failed">%1$s: could not be read</string>
|
||||
<string name="chat_sent_image">Attached image</string>
|
||||
<string name="consent_secret_hint">The secret the agent asked for</string>
|
||||
<string name="consent_submit">Send secret</string>
|
||||
<string name="consent_approve">Approve</string>
|
||||
@@ -148,5 +159,10 @@
|
||||
<string name="sessions_group_other">Other keys</string>
|
||||
|
||||
<string name="chat_agent">rhema</string>
|
||||
|
||||
<!-- The model listing's vision verdict, suffixed onto the model id;
|
||||
absent means unknown, so there is no third string. -->
|
||||
<string name="vision_badge">" ◉ vision"</string>
|
||||
<string name="vision_badge_absent">" no vision"</string>
|
||||
</resources>
|
||||
|
||||
@@ -3,4 +3,7 @@
|
||||
hand-off; FileProvider maps files under it to content:// uris. -->
|
||||
<paths>
|
||||
<cache-path name="attachments" path="attachments/" />
|
||||
<!-- Camera shots stage here before a send: TakePicture writes straight
|
||||
into the FileProvider-mapped content:// uri (no CAMERA permission). -->
|
||||
<cache-path name="camera" path="camera/" />
|
||||
</paths>
|
||||
Reference in new issue
Block a user