From 5b615a98ae2b0ea1621c71ef1e079575a56263c0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 21:03:03 +0000 Subject: [PATCH 1/6] Open chat visuals full screen on iOS and Android (Remote revision 34) Phones keep the inline snapshot; with the new chat-visual-viewer-v1 token, tapping it opens the visual in a locked-down web view. The Mac serves GET /chats/{chatId}/visuals/{visualId}/document: one self-contained HTML document built from a new viewer bundle (render_ui via AidenUiBlock, render_artifact as the sandboxed guest with only the libraries it uses). Follow-ups and links go to the phone, which confirms; prompts only fill the composer. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01KPgvZHFWvJ95VRjuu4p9LL --- .memory/inline-generative-ui.md | 8 + android/app/proguard-rules.pro | 6 + .../features/chat/AidenChatDetailScreen.kt | 55 +- .../features/chat/AidenChatViewModel.kt | 40 ++ .../chat/AidenMessageImageAttachments.kt | 30 +- .../features/chat/AidenVisualViewerModel.kt | 101 ++++ .../features/chat/AidenVisualViewerScreen.kt | 553 ++++++++++++++++++ .../AidenOnTheGo/models/AidenInstallation.kt | 4 + .../models/AidenVisualDocument.kt | 52 ++ .../networking/AidenRemoteClient.kt | 80 +++ .../persistence/AidenVisualDocumentCache.kt | 167 ++++++ .../protocol/AidenRemoteProtocol.kt | 3 + android/app/src/main/res/values/strings.xml | 17 + .../AidenVisualDocumentClientTest.kt | 205 +++++++ .../chat/AidenVisualViewerModelTest.kt | 108 ++++ .../AidenVisualDocumentCacheTest.kt | 149 +++++ docs/aiden-remote-api-v1.md | 9 + docs/plans/README.md | 1 + docs/plans/phone-visual-viewer-plan.md | 52 ++ ios/AidenOnTheGo.xcodeproj/project.pbxproj | 12 + ios/AidenOnTheGo/Config/AidenAppearance.swift | 9 + .../Features/Remote/AidenChatFeature.swift | 169 +++++- .../Remote/AidenRemoteCoordinator.swift | 1 + .../Features/Remote/AidenVisualDocument.swift | 305 ++++++++++ .../Features/Remote/AidenVisualViewer.swift | 373 ++++++++++++ .../Networking/AidenRemoteClient.swift | 55 ++ .../Networking/AidenRemoteContract.swift | 14 + ios/AidenOnTheGoTests/AidenChatTests.swift | 105 ++++ .../AidenRemoteClientTests.swift | 135 +++++ .../AidenRemotePhase0Tests.swift | 2 +- .../AidenVisualViewerTests.swift | 285 +++++++++ ios/CHANGELOG.md | 2 + main/services/aiden-remote-chats.test.ts | 74 +++ main/services/aiden-remote-chats.ts | 54 ++ .../aiden-remote-operation-contract.test.ts | 2 +- main/services/aiden-remote-protocol.test.ts | 3 +- main/services/aiden-remote-protocol.ts | 14 +- main/services/aiden-remote-router.test.ts | 96 ++- main/services/aiden-remote-router.ts | 68 ++- main/services/aiden-remote-service-main.ts | 13 + main/services/generative-ui-html.ts | 46 +- main/services/visual-viewer-document.test.ts | 120 ++++ main/services/visual-viewer-document.ts | 158 +++++ main/services/visual-viewer-main.ts | 46 ++ main/windows/window-paths.ts | 5 + package.json | 7 +- .../aiden-remote/v1/fixtures/contract.json | 4 +- protocol/aiden-remote/v1/openapi.json | 71 +++ .../components/aiden-ui/aiden-ui-block.tsx | 14 +- renderer/lib/link-opener.tsx | 22 + renderer/shared/visual-viewer.ts | 85 +++ renderer/visual-viewer/main.tsx | 198 +++++++ scripts/ci-test-registry.json | 1 + tests/generative-ui/visual-viewer.spec.ts | 141 +++++ tsconfig.json | 3 +- vite.visual-viewer.config.ts | 62 ++ 56 files changed, 4365 insertions(+), 49 deletions(-) create mode 100644 android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenVisualViewerModel.kt create mode 100644 android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenVisualViewerScreen.kt create mode 100644 android/app/src/main/java/sbtbiswas/AidenOnTheGo/models/AidenVisualDocument.kt create mode 100644 android/app/src/main/java/sbtbiswas/AidenOnTheGo/persistence/AidenVisualDocumentCache.kt create mode 100644 android/app/src/test/java/sbtbiswas/AidenOnTheGo/AidenVisualDocumentClientTest.kt create mode 100644 android/app/src/test/java/sbtbiswas/AidenOnTheGo/features/chat/AidenVisualViewerModelTest.kt create mode 100644 android/app/src/test/java/sbtbiswas/AidenOnTheGo/persistence/AidenVisualDocumentCacheTest.kt create mode 100644 docs/plans/phone-visual-viewer-plan.md create mode 100644 ios/AidenOnTheGo/Features/Remote/AidenVisualDocument.swift create mode 100644 ios/AidenOnTheGo/Features/Remote/AidenVisualViewer.swift create mode 100644 ios/AidenOnTheGoTests/AidenVisualViewerTests.swift create mode 100644 main/services/visual-viewer-document.test.ts create mode 100644 main/services/visual-viewer-document.ts create mode 100644 main/services/visual-viewer-main.ts create mode 100644 renderer/shared/visual-viewer.ts create mode 100644 renderer/visual-viewer/main.tsx create mode 100644 tests/generative-ui/visual-viewer.spec.ts create mode 100644 vite.visual-viewer.config.ts diff --git a/.memory/inline-generative-ui.md b/.memory/inline-generative-ui.md index 57168aec4..f3bd22ff4 100644 --- a/.memory/inline-generative-ui.md +++ b/.memory/inline-generative-ui.md @@ -40,10 +40,18 @@ Spec: `docs/superpowers/specs/2026-10-08-inline-generative-ui-design.md`. Plan: - **Phones.** iOS (`AidenChatVisual`, `AidenChatVisualPresentation` in `Models/AidenChat.swift`) and Android (`AidenChatVisual`, `AidenChatVisualListSerializer` in `models/AidenChat.kt`, row helpers in `AidenChatDetailScreen.kt`) decode visuals per entry and place each after the tool step that drew it, splitting a same-offset tool group at that step. A snapshot counts only when exactly one PNG/JPEG with that id is on the message. An html visual without one gets no row; its can't-view card stands for it. iOS keeps an unknown kind as `.other`; desktop and Android drop it. Live streaming rows never carry visuals. - **Remote turns.** `remoteGenerationSurface` follows the Mac's Appearance setting (injected `inlineVisuals()` reading `configStore` appearance) and sets `visualAudience: "remote"`, which adds a system-prompt note that readers see still images. +- **Phone viewer (contract revision 34, provisional; `chat-visual-viewer-v1`).** Owner chose tap to open (2026-10-16). Plan: `docs/plans/phone-visual-viewer-plan.md`. + - `GET /chats/{chatId}/visuals/{visualId}/document?scheme&theme` returns one self-contained HTML document (`main/services/visual-viewer-document.ts`, Electron-free). Visual entries gained no key: their parsers are closed on every client, so a new key would drop all visuals on older phones. + - The viewer page is `renderer/visual-viewer/main.tsx`, built by `vite.visual-viewer.config.ts` (lib IIFE, assets inlined, KaTeX woff/ttf stripped) into `build/renderer/visual-viewer/` after the main renderer build. The feature is announced only when those files exist (`visual-viewer-main.ts`). + - Theme: the page applies the phone's preset with `resolveThemeTokens` (no storage; opaque origins throw on localStorage). HTML guests start with light defaults, so the iframe stays hidden until the guest's `ready` gets a theme message. + - HTML guests inline only the host libraries their source names (`Chart`, `Plotly`, `katex`). `inlineGenerativeUiHostLibraries` is shared with export (and the CLI) and uses a function replacement so `$&` in library source stays literal. + - Native channel: iOS `aidenVisual` handler, Android `AidenVisualBridge`; messages `ready`, `prompt`, `openUrl`. `AidenUiBlock` takes `onFollowUp` so the viewer hands prompts to the phone instead of drawing its own Send chip; the phone confirms and only fills the composer. Links use `ExternalLinkOpenerProvider`. + ## Tests - `npm run test:generative-ui`: units, the vendor-script test, and Chromium containment (bridge, draft, shrink, rem). - happy-dom component tests: `renderer/components/html-artifact-frame.test.tsx`, `message-list-visuals.test.tsx`. Compare DOM nodes as booleans; printing a happy-dom node in an assert diff hangs. - e2e: `settings-unification.spec.ts` "inline visuals preference…", and `inline-visuals.spec.ts` (fake model scripts `render_ui` + `render_artifact` via `enqueueToolScenario`, then checks both snapshots are PNG attachments and the capture window stays hidden). +- Phone viewer: `main/services/visual-viewer-document.test.ts`, the visual-document cases in `aiden-remote-chats.test.ts` and `aiden-remote-router.test.ts`, and `tests/generative-ui/visual-viewer.spec.ts` (Chromium, network blocked; needs `npm run build:visual-viewer`, which `pretest:generative-ui` runs). - Phase 3: `main/services/visual-snapshot-core.test.ts`, `renderer/shared/visual-snapshots.test.ts`, `main/services/visual-snapshot-store.test.ts`, `renderer/components/message-attachments-snapshots.test.tsx`, and the visuals cases in `aiden-remote-chats.test.ts` / `aiden-remote-protocol.test.ts`. - Phase 2: `renderer/shared/aiden-ui/*.test.ts` (parser vectors including every streaming prefix, `fixtures/expressions.json`, the compile corpus in `fixtures/markup/*.aum`), `renderer/components/ui-primitives.test.tsx`, `renderer/components/aiden-ui/aiden-ui-block.test.tsx` (Chart.js stubbed), `main/services/aiden-ui-{tool,draft}.test.ts`, `main/handlers/chat-ui-visual-state.test.ts`, `renderer/lib/ui-visual-transcript.test.ts`. The CLI needs `npm ci` in `packages/cli` before `npm run build && npm test`. diff --git a/android/app/proguard-rules.pro b/android/app/proguard-rules.pro index 2954e3830..70db1b933 100644 --- a/android/app/proguard-rules.pro +++ b/android/app/proguard-rules.pro @@ -20,3 +20,9 @@ -dontwarn org.bouncycastle.jsse.** -dontwarn org.conscrypt.** -dontwarn org.openjsse.** + +# The interactive visual viewer's WebView bridge (contract revision 34) is called +# by name from JavaScript, so its annotated methods must keep their names. +-keepclassmembers class sbtbiswas.AidenOnTheGo.features.chat.** { + @android.webkit.JavascriptInterface ; +} diff --git a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenChatDetailScreen.kt b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenChatDetailScreen.kt index 089eb5bf6..053f2891c 100644 --- a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenChatDetailScreen.kt +++ b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenChatDetailScreen.kt @@ -248,6 +248,8 @@ fun AidenChatDetailScreen( var requestedNotificationPermission by rememberSaveable { mutableStateOf(false) } var progressSheet by rememberSaveable { mutableStateOf(null) } var selectTextFor by remember { mutableStateOf(null) } + var visualViewerRequest by remember(chatId) { mutableStateOf(null) } + var visualSnapshotGallery by remember(chatId) { mutableStateOf(null) } val currentDraft by rememberUpdatedState(draft) val currentVoiceMode by rememberUpdatedState(voiceInputMode) @@ -905,6 +907,11 @@ fun AidenChatDetailScreen( val onAskAboutMessage: ((String) -> Unit)? = if (viewModel.isReadOnlyPresentation) null else remember(viewModel) { { text: String -> viewModel.askAbout(text); Unit } } val onSelectMessageText = remember { { text: String -> selectTextFor = text } } + // Revision 34: a tapped snapshot opens the interactive viewer when the Mac serves it. + val canOpenInteractiveVisuals = remember(serverInfo, viewModel) { viewModel.canOpenInteractiveVisuals } + val openVisualViewer = remember { { request: AidenVisualViewerRequest -> visualViewerRequest = request } } + val onOpenVisual: ((AidenVisualViewerRequest) -> Unit)? = + if (canOpenInteractiveVisuals) openVisualViewer else null val canForkMessages = viewModel.canFork && !isForking val canForkMessagesWithSummary = canForkMessages && viewModel.canForkWithSummary val forkLineage = chat?.forkedFrom @@ -1066,7 +1073,8 @@ fun AidenChatDetailScreen( isPinnedToStreamingEdge = isPinnedToStreamingEdge, imageArrivalRevealGenerationId = message.timeline?.generationId ?.takeIf { it in imageArrivalRevealIds }, - onImageArrivalRevealed = viewModel::finishImageArrivalReveal + onImageArrivalRevealed = viewModel::finishImageArrivalReveal, + onOpenVisual = onOpenVisual ) } // The summary stands where the copied conversation ends. @@ -1180,6 +1188,35 @@ fun AidenChatDetailScreen( ) } + visualViewerRequest?.let { request -> + AidenVisualViewerScreen( + request = request, + chatId = chatId, + hostIdentity = viewModel.visualDocumentHostIdentity, + fetch = viewModel::visualDocument, + onShowImage = request.snapshot?.let { snapshot -> + { + visualViewerRequest = null + visualSnapshotGallery = snapshot + } + }, + onAddToMessage = if (viewModel.isReadOnlyPresentation) null else ({ text: String -> + visualViewerRequest = null + viewModel.addVisualPrompt(text) + Unit + }), + onDismiss = { visualViewerRequest = null } + ) + } + visualSnapshotGallery?.let { snapshot -> + AidenAttachmentGallery( + attachments = listOf(snapshot), + initialPage = 0, + loadData = viewModel::attachmentImageData, + onDismiss = { visualSnapshotGallery = null } + ) + } + forkWithSummaryMessageId?.let { messageId -> AidenForkSummaryDialog( palette = palette, @@ -1343,7 +1380,9 @@ private fun AssistantMessageRow( isPinnedToStreamingEdge: () -> Boolean = { false }, /** Set while this message's generated image should play the one-shot arrival reveal. */ imageArrivalRevealGenerationId: String? = null, - onImageArrivalRevealed: (String) -> Unit = {} + onImageArrivalRevealed: (String) -> Unit = {}, + /** Opens a visual in the interactive viewer; null keeps snapshot taps on the image viewer. */ + onOpenVisual: ((AidenVisualViewerRequest) -> Unit)? = null ) { val projection = if (isBotChat) { AidenBotReplyProjection.resolve(message.text, message.timeline, isActive = false) @@ -1366,7 +1405,8 @@ private fun AssistantMessageRow( visual = visual, attachments = message.attachments.orEmpty(), loadAttachmentImage = loadAttachmentImage, - palette = palette + palette = palette, + onOpenInteractive = onOpenVisual ) } @@ -1607,7 +1647,9 @@ internal fun AidenVisualRow( attachments: List, loadAttachmentImage: suspend (AidenMessageAttachment) -> ByteArray?, palette: sbtbiswas.AidenOnTheGo.config.AidenPalette, - modifier: Modifier = Modifier + modifier: Modifier = Modifier, + /** Set when the Mac serves interactive visuals (revision 34); tapping the snapshot opens it. */ + onOpenInteractive: ((AidenVisualViewerRequest) -> Unit)? = null ) { val display = remember(visual, attachments) { aidenVisualDisplay(visual, attachments) } val snapshot = display.snapshot @@ -1617,7 +1659,10 @@ internal fun AidenVisualRow( attachment = snapshot, title = visual.title, wide = visual.layout == AidenChatVisual.LAYOUT_WIDE, - loadData = loadAttachmentImage + loadData = loadAttachmentImage, + onOpenInteractive = onOpenInteractive?.let { open -> + { open(AidenVisualViewerRequest(visual = visual, snapshot = snapshot)) } + } ) display.text?.let { AidenVisualFallbackText(text = it, visualId = visual.id, palette = palette) } } diff --git a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenChatViewModel.kt b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenChatViewModel.kt index 80594ce60..e1a3f2068 100644 --- a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenChatViewModel.kt +++ b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenChatViewModel.kt @@ -1178,6 +1178,46 @@ class AidenChatViewModel( return true } + /** + * The Mac announced `chat-visual-viewer-v1` (revision 34), so a tapped visual + * snapshot opens the interactive viewer instead of the image viewer. + */ + val canOpenInteractiveVisuals: Boolean + get() = instanceId.isNotEmpty() && AidenVisualViewerGate.isAvailable( + coordinator.serverInfo.value?.features, + coordinator.installationStore.activeInstallation + ?.takeIf { it.instanceId == instanceId && it.deviceId == deviceId } + ?.serverFeatures + ) + + /** Names this pairing in visual document cache keys. */ + val visualDocumentHostIdentity: String + get() = "$instanceId\u001f$deviceId" + + /** The viewer page for [visualId] from the bound Mac (revision 34). */ + suspend fun visualDocument( + visualId: String, + scheme: AidenVisualDocumentScheme, + theme: String + ): String { + val client = activeClient() ?: throw AidenRemoteClientException.Disconnected() + val html = client.visualDocument(chatId, visualId, scheme, theme) + if (activeClient() !== client) throw AidenRemoteClientException.Disconnected() + return html + } + + /** + * Appends a follow-up a visual asked for, after the user confirmed it, to the + * composer draft. It is never sent from here. Returns false when the chat is + * read-only or the text is blank. + */ + fun addVisualPrompt(text: String): Boolean { + if (isReadOnlyPresentation) return false + val next = AidenVisualPromptDraft.appended(_draft.value, text) ?: return false + updateDraft(next) + return true + } + fun updateDraft(text: String) { _draft.value = text draftWriter?.schedule(text) diff --git a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenMessageImageAttachments.kt b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenMessageImageAttachments.kt index d08b8ce4c..b19a1b9f5 100644 --- a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenMessageImageAttachments.kt +++ b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenMessageImageAttachments.kt @@ -259,7 +259,9 @@ internal object AidenVisualSnapshotFrame { /** * A visual's snapshot, described by the visual's [title] rather than the - * attachment's file name. Tapping opens the same full-screen viewer as photos. + * attachment's file name. Tapping opens the same full-screen viewer as photos, + * or, when the Mac serves interactive visuals ([onOpenInteractive], revision 34), + * the interactive viewer; the image viewer then stays one accessibility action away. */ @Composable internal fun AidenVisualSnapshotImage( @@ -267,7 +269,8 @@ internal fun AidenVisualSnapshotImage( title: String, wide: Boolean, loadData: suspend (AidenMessageAttachment) -> ByteArray?, - modifier: Modifier = Modifier + modifier: Modifier = Modifier, + onOpenInteractive: (() -> Unit)? = null ) { var galleryOpen by remember { mutableStateOf(false) } // An image this session already decoded sizes the row on its first frame. @@ -279,6 +282,15 @@ internal fun AidenVisualSnapshotImage( } val snapshotState = stringResource(R.string.chat_visual_snapshot_description) val openLabel = stringResource(R.string.chat_visual_snapshot_open) + val openInteractiveLabel = stringResource(R.string.chat_visual_open_interactive) + val interactiveSemantics = if (onOpenInteractive != null) { + Modifier.semantics { + customActions = listOf( + CustomAccessibilityAction(openInteractiveLabel) { onOpenInteractive(); true }, + CustomAccessibilityAction(openLabel) { galleryOpen = true; true } + ) + } + } else Modifier AidenAttachmentImage( attachment = attachment, maximumPixelSize = AidenVisualSnapshotFrame.MAXIMUM_PIXEL_SIZE, @@ -289,7 +301,17 @@ internal fun AidenVisualSnapshotImage( .fillMaxWidth() .aspectRatio(AidenVisualSnapshotFrame.aspectRatio(imageSize?.first, imageSize?.second, wide)) .semantics { stateDescription = snapshotState } - .clickable(onClickLabel = openLabel, role = Role.Button) { galleryOpen = true }, + .then(interactiveSemantics) + .clickable( + onClickLabel = if (onOpenInteractive != null) openInteractiveLabel else openLabel, + role = Role.Button + ) { + if (onOpenInteractive != null) { + onOpenInteractive() + } else { + galleryOpen = true + } + }, alignment = Alignment.CenterStart, imageCornerRadius = 16.dp ) @@ -572,7 +594,7 @@ private fun AidenAttachmentImage( } @Composable -private fun AidenAttachmentGallery( +internal fun AidenAttachmentGallery( attachments: List, initialPage: Int, loadData: suspend (AidenMessageAttachment) -> ByteArray?, diff --git a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenVisualViewerModel.kt b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenVisualViewerModel.kt new file mode 100644 index 000000000..cceca3c33 --- /dev/null +++ b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenVisualViewerModel.kt @@ -0,0 +1,101 @@ +package sbtbiswas.AidenOnTheGo.features.chat + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import java.net.URI + +/** + * One message the visual viewer page sends through `window.AidenVisualBridge.postMessage` + * (contract revision 34). The set is closed: anything else is ignored. + */ +sealed interface AidenVisualBridgeMessage { + /** The visual has drawn; the loading indicator can go. */ + data object Ready : AidenVisualBridgeMessage + + /** The visual asks for a follow-up. Native confirms before adding it to the composer. */ + data class Prompt(val text: String) : AidenVisualBridgeMessage + + /** A `render_ui` `openUrl` action. Native confirms before opening it in the browser. */ + data class OpenUrl(val url: String) : AidenVisualBridgeMessage + + companion object { + /** The page bounds prompts to this many characters; native enforces it again. */ + const val MAXIMUM_PROMPT_LENGTH = 2_000 + const val MAXIMUM_URL_LENGTH = 2_048 + /** Upper bound on one raw bridge message, far above any valid one. */ + const val MAXIMUM_MESSAGE_LENGTH = 16 * 1_024 + + private val json = Json { isLenient = false } + + /** The message [raw] carries, or null for anything outside the closed set. */ + fun parse(raw: String): AidenVisualBridgeMessage? { + if (raw.length > MAXIMUM_MESSAGE_LENGTH) return null + val root = try { + json.parseToJsonElement(raw) as? JsonObject + } catch (_: Exception) { + null + } ?: return null + return when (root.stringValue("type")) { + "ready" -> Ready + "prompt" -> { + val text = root.stringValue("text")?.trim() ?: return null + if (text.isEmpty() || text.length > MAXIMUM_PROMPT_LENGTH) return null + Prompt(text) + } + "openUrl" -> { + val url = root.stringValue("url") ?: return null + if (!isOpenableUrl(url)) return null + OpenUrl(url) + } + else -> null + } + } + + /** Only an absolute `https:` URL with a host and no user info may be opened. */ + fun isOpenableUrl(url: String): Boolean { + if (url.isEmpty() || url.length > MAXIMUM_URL_LENGTH) return false + if (url.any { it.isWhitespace() || Character.isISOControl(it) }) return false + val uri = try { URI(url) } catch (_: Exception) { return false } + return uri.isAbsolute && + uri.scheme == "https" && + uri.rawUserInfo == null && + !uri.host.isNullOrEmpty() + } + + private fun JsonObject.stringValue(key: String): String? { + val primitive = this[key] as? JsonPrimitive ?: return null + return if (primitive.isString) primitive.content else null + } + } +} + +/** How a confirmed visual prompt joins the composer draft: appended, never sent. */ +object AidenVisualPromptDraft { + fun appended(existingDraft: String, prompt: String): String? { + val text = prompt.trim() + if (text.isEmpty()) return null + val existing = existingDraft.trimEnd() + return if (existing.isEmpty()) text else "$existing\n\n$text" + } +} + +/** + * Native rate limit for prompts and links, matching the page's one per three + * seconds. JavaScript interfaces are reachable from every frame of the page, so + * native does not rely on the page's own bound. + */ +class AidenVisualBridgeThrottle( + private val intervalMillis: Long = 3_000L, + private val now: () -> Long +) { + private var lastAccepted: Long? = null + + fun tryAcquire(): Boolean { + val time = now() + val last = lastAccepted + if (last != null && time - last < intervalMillis && time >= last) return false + lastAccepted = time + return true + } +} diff --git a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenVisualViewerScreen.kt b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenVisualViewerScreen.kt new file mode 100644 index 000000000..3ec0d573b --- /dev/null +++ b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/features/chat/AidenVisualViewerScreen.kt @@ -0,0 +1,553 @@ +package sbtbiswas.AidenOnTheGo.features.chat + +import android.annotation.SuppressLint +import android.content.Context +import android.content.Intent +import android.net.Uri +import android.os.Handler +import android.os.Looper +import android.os.Message +import android.os.SystemClock +import android.webkit.GeolocationPermissions +import android.webkit.JavascriptInterface +import android.webkit.JsPromptResult +import android.webkit.JsResult +import android.webkit.PermissionRequest +import android.webkit.RenderProcessGoneDetail +import android.webkit.ValueCallback +import android.webkit.WebChromeClient +import android.webkit.WebResourceRequest +import android.webkit.WebResourceResponse +import android.webkit.WebSettings +import android.webkit.WebView +import android.webkit.WebViewClient +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.heightIn +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.systemBarsPadding +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.verticalScroll +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Close +import androidx.compose.material.icons.filled.Image +import androidx.compose.material3.AlertDialog +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.Immutable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.key +import androidx.compose.runtime.mutableIntStateOf +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberUpdatedState +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.semantics.contentDescription +import androidx.compose.ui.semantics.heading +import androidx.compose.ui.semantics.liveRegion +import androidx.compose.ui.semantics.LiveRegionMode +import androidx.compose.ui.semantics.semantics +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import androidx.compose.ui.viewinterop.AndroidView +import androidx.compose.ui.window.Dialog +import androidx.compose.ui.window.DialogProperties +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext +import sbtbiswas.AidenOnTheGo.R +import sbtbiswas.AidenOnTheGo.models.AidenChatVisual +import sbtbiswas.AidenOnTheGo.models.AidenMessageAttachment +import sbtbiswas.AidenOnTheGo.models.AidenVisualDocumentRequest +import sbtbiswas.AidenOnTheGo.models.AidenVisualDocumentScheme +import sbtbiswas.AidenOnTheGo.persistence.AidenVisualDocumentCache +import sbtbiswas.AidenOnTheGo.persistence.AidenVisualDocumentKey +import sbtbiswas.AidenOnTheGo.persistence.AidenVisualDocumentLoad +import sbtbiswas.AidenOnTheGo.persistence.AidenVisualDocumentLoader +import sbtbiswas.AidenOnTheGo.protocol.AidenRemoteClientException +import sbtbiswas.AidenOnTheGo.ui.theme.AidenDialogConfirmButton +import sbtbiswas.AidenOnTheGo.ui.theme.AidenDialogDismissButton +import sbtbiswas.AidenOnTheGo.ui.theme.AidenShape +import sbtbiswas.AidenOnTheGo.ui.theme.AidenTheme +import sbtbiswas.AidenOnTheGo.ui.theme.AidenTonalButton +import java.io.ByteArrayInputStream +import java.io.File + +/** A visual the user tapped, with the snapshot the viewer can fall back to. */ +@Immutable +data class AidenVisualViewerRequest( + val visual: AidenChatVisual, + val snapshot: AidenMessageAttachment? +) + +private sealed interface AidenVisualViewerContent { + data object Loading : AidenVisualViewerContent + data class Document(val html: String, val stale: Boolean) : AidenVisualViewerContent + data class Failed(val unavailable: Boolean) : AidenVisualViewerContent +} + +/** The single name the viewer page posts to (`window.AidenVisualBridge.postMessage`). */ +private const val AIDEN_VISUAL_BRIDGE_NAME = "AidenVisualBridge" + +/** Without a `ready` message the indicator still clears, so a quiet page is never hidden. */ +private const val READY_FALLBACK_MILLIS = 10_000L + +/** + * A visual opened full screen (contract revision 34). The Mac's self-contained + * viewer page runs in a locked-down [WebView]: JavaScript on, no file, content, + * storage, window or navigation access, and one message channel, which only ever + * asks native to confirm a follow-up or a link. When the Mac cannot be reached the + * last cached copy is shown with a note that it may be out of date. + */ +@Composable +internal fun AidenVisualViewerScreen( + request: AidenVisualViewerRequest, + chatId: String, + hostIdentity: String, + fetch: suspend (visualId: String, scheme: AidenVisualDocumentScheme, theme: String) -> String, + onShowImage: (() -> Unit)?, + /** Appends a confirmed follow-up to the composer and closes; null ignores prompts (read-only). */ + onAddToMessage: ((String) -> Unit)?, + onDismiss: () -> Unit +) { + val palette = AidenTheme.palette + val scheme = AidenVisualDocumentScheme.forDarkTheme(palette.isDark) + val theme = AidenVisualDocumentRequest.themeId(AidenTheme.config.preset) + val context = LocalContext.current + val cache = remember(context) { AidenVisualDocumentCacheStore.get(context) } + val currentFetch by rememberUpdatedState(fetch) + val visualId = request.visual.id + + var content by remember(visualId) { mutableStateOf(AidenVisualViewerContent.Loading) } + var ready by remember(visualId) { mutableStateOf(false) } + var attempt by remember(visualId) { mutableIntStateOf(0) } + var rendererGeneration by remember(visualId) { mutableIntStateOf(0) } + var rendererGone by remember(visualId) { mutableStateOf(false) } + var pendingPrompt by remember(visualId) { mutableStateOf(null) } + var pendingUrl by remember(visualId) { mutableStateOf(null) } + val throttle = remember(visualId) { AidenVisualBridgeThrottle { SystemClock.elapsedRealtime() } } + + LaunchedEffect(visualId, scheme, theme, attempt) { + content = AidenVisualViewerContent.Loading + ready = false + rendererGone = false + val key = AidenVisualDocumentKey(hostIdentity, chatId, visualId, scheme.wire, theme) + val loader = AidenVisualDocumentLoader( + cache = cache, + fetch = { currentFetch(visualId, scheme, theme) }, + isNotFound = { error -> error is AidenRemoteClientException.Server && error.statusCode == 404 } + ) + content = when (val result = withContext(Dispatchers.IO) { loader.load(key) }) { + is AidenVisualDocumentLoad.Fresh -> AidenVisualViewerContent.Document(result.html, stale = false) + is AidenVisualDocumentLoad.Cached -> AidenVisualViewerContent.Document(result.html, stale = true) + is AidenVisualDocumentLoad.Failed -> AidenVisualViewerContent.Failed(result.unavailable) + } + } + LaunchedEffect(content, rendererGeneration) { + if (content is AidenVisualViewerContent.Document) { + delay(READY_FALLBACK_MILLIS) + ready = true + } + } + + val onBridgeMessage: (AidenVisualBridgeMessage) -> Unit = { message -> + when (message) { + AidenVisualBridgeMessage.Ready -> { + ready = true + } + is AidenVisualBridgeMessage.Prompt -> { + // A newer prompt replaces one still waiting for an answer. + if (onAddToMessage != null && throttle.tryAcquire()) pendingPrompt = message.text + } + is AidenVisualBridgeMessage.OpenUrl -> { + if (throttle.tryAcquire()) pendingUrl = message.url + } + } + Unit + } + + Dialog( + onDismissRequest = onDismiss, + properties = DialogProperties(usePlatformDefaultWidth = false, decorFitsSystemWindows = false) + ) { + Surface(color = palette.canvas, contentColor = palette.foreground, modifier = Modifier.fillMaxSize()) { + Column(modifier = Modifier.fillMaxSize().systemBarsPadding()) { + Row( + modifier = Modifier.fillMaxWidth().padding(horizontal = 4.dp, vertical = 4.dp), + verticalAlignment = Alignment.CenterVertically + ) { + IconButton(onClick = onDismiss) { + Icon(Icons.Default.Close, contentDescription = stringResource(R.string.chat_visual_viewer_close)) + } + Text( + text = request.visual.title, + style = MaterialTheme.typography.titleMedium, + color = palette.foreground, + maxLines = 1, + overflow = TextOverflow.Ellipsis, + modifier = Modifier + .weight(1f) + .padding(horizontal = 8.dp) + .semantics { heading() } + ) + if (onShowImage != null) { + IconButton(onClick = onShowImage) { + Icon(Icons.Default.Image, contentDescription = stringResource(R.string.chat_visual_viewer_show_image)) + } + } + } + + val current = content + if (current is AidenVisualViewerContent.Document && current.stale) { + AidenVisualViewerNotice( + text = stringResource(R.string.chat_visual_viewer_stale), + onRetry = { attempt++ } + ) + } + + Box(modifier = Modifier.weight(1f).fillMaxWidth()) { + when { + current is AidenVisualViewerContent.Failed -> AidenVisualViewerMessage( + text = stringResource( + if (current.unavailable) R.string.chat_visual_viewer_unavailable + else R.string.chat_visual_viewer_failed + ), + onRetry = { attempt++ } + ) + current is AidenVisualViewerContent.Document && rendererGone -> AidenVisualViewerMessage( + text = stringResource(R.string.chat_visual_viewer_stopped), + onRetry = { + rendererGone = false + ready = false + rendererGeneration++ + } + ) + current is AidenVisualViewerContent.Document -> key(current.html, rendererGeneration) { + AidenVisualWebView( + html = current.html, + onMessage = onBridgeMessage, + onRendererGone = { rendererGone = true }, + modifier = Modifier.fillMaxSize() + ) + } + } + val loading = current is AidenVisualViewerContent.Loading || + (current is AidenVisualViewerContent.Document && !rendererGone && !ready) + if (loading) { + val loadingLabel = stringResource(R.string.chat_visual_viewer_loading) + CircularProgressIndicator( + color = palette.secondary, + modifier = Modifier + .align(Alignment.Center) + .size(28.dp) + .semantics { contentDescription = loadingLabel } + ) + } + } + } + } + + pendingPrompt?.let { prompt -> + AlertDialog( + onDismissRequest = { pendingPrompt = null }, + title = { Text(stringResource(R.string.chat_visual_prompt_title)) }, + text = { + Column( + modifier = Modifier.heightIn(max = 280.dp).verticalScroll(rememberScrollState()), + verticalArrangement = Arrangement.spacedBy(8.dp) + ) { + Text( + text = stringResource(R.string.chat_visual_prompt_body), + style = MaterialTheme.typography.bodyMedium, + color = palette.secondary + ) + Text( + text = stringResource(R.string.chat_visual_prompt_quote, prompt), + style = MaterialTheme.typography.bodyLarge, + color = palette.foreground + ) + } + }, + confirmButton = { + AidenDialogConfirmButton( + text = stringResource(R.string.chat_visual_prompt_add), + onClick = { + pendingPrompt = null + onAddToMessage?.invoke(prompt) + } + ) + }, + dismissButton = { + AidenDialogDismissButton( + text = stringResource(R.string.chat_visual_prompt_dismiss), + onClick = { pendingPrompt = null } + ) + }, + shape = AidenShape.Dialog, + containerColor = palette.raised + ) + } + + pendingUrl?.let { url -> + AlertDialog( + onDismissRequest = { pendingUrl = null }, + title = { Text(stringResource(R.string.chat_visual_link_title)) }, + text = { + Text( + text = url, + style = MaterialTheme.typography.bodyMedium, + color = palette.secondary, + maxLines = 6, + overflow = TextOverflow.Ellipsis + ) + }, + confirmButton = { + AidenDialogConfirmButton( + text = stringResource(R.string.chat_visual_link_open), + onClick = { + pendingUrl = null + openAidenVisualLink(context, url) + } + ) + }, + dismissButton = { AidenDialogDismissButton(onClick = { pendingUrl = null }) }, + shape = AidenShape.Dialog, + containerColor = palette.raised + ) + } + } +} + +@Composable +private fun AidenVisualViewerNotice(text: String, onRetry: () -> Unit) { + val palette = AidenTheme.palette + Surface( + color = MaterialTheme.colorScheme.surfaceContainerHigh, + shape = MaterialTheme.shapes.medium, + modifier = Modifier.fillMaxWidth().padding(horizontal = 12.dp, vertical = 4.dp) + ) { + Row( + modifier = Modifier.padding(start = 12.dp, end = 4.dp, top = 4.dp, bottom = 4.dp), + verticalAlignment = Alignment.CenterVertically + ) { + Text( + text = text, + style = MaterialTheme.typography.bodySmall, + color = palette.secondary, + modifier = Modifier + .weight(1f) + .semantics { liveRegion = LiveRegionMode.Polite } + ) + AidenTonalButton(text = stringResource(R.string.action_retry), onClick = onRetry) + } + } +} + +@Composable +private fun AidenVisualViewerMessage(text: String, onRetry: () -> Unit) { + val palette = AidenTheme.palette + Column( + modifier = Modifier.fillMaxSize().padding(horizontal = 24.dp), + verticalArrangement = Arrangement.spacedBy(16.dp, Alignment.CenterVertically), + horizontalAlignment = Alignment.CenterHorizontally + ) { + Text( + text = text, + style = MaterialTheme.typography.bodyMedium, + color = palette.secondary, + textAlign = TextAlign.Center, + modifier = Modifier.semantics { liveRegion = LiveRegionMode.Polite } + ) + AidenTonalButton(text = stringResource(R.string.action_retry), onClick = onRetry) + } +} + +@Composable +private fun AidenVisualWebView( + html: String, + onMessage: (AidenVisualBridgeMessage) -> Unit, + onRendererGone: () -> Unit, + modifier: Modifier = Modifier +) { + val currentOnMessage by rememberUpdatedState(onMessage) + val currentOnRendererGone by rememberUpdatedState(onRendererGone) + val bridge = remember { AidenVisualBridgeInterface { message -> currentOnMessage(message) } } + DisposableEffect(bridge) { + onDispose { bridge.close() } + } + AndroidView( + factory = { context -> + createAidenVisualWebView(context, bridge) { currentOnRendererGone() }.also { webView -> + // No base URL: the page gets an opaque origin and nothing to resolve against. + webView.loadDataWithBaseURL(null, html, "text/html", "utf-8", null) + } + }, + modifier = modifier, + onRelease = { webView -> + webView.stopLoading() + webView.removeJavascriptInterface(AIDEN_VISUAL_BRIDGE_NAME) + webView.webChromeClient = null + webView.destroy() + } + ) +} + +@SuppressLint("SetJavaScriptEnabled", "JavascriptInterface") +private fun createAidenVisualWebView( + context: Context, + bridge: AidenVisualBridgeInterface, + onRendererGone: () -> Unit +): WebView = WebView(context).apply { + settings.apply { + javaScriptEnabled = true + allowFileAccess = false + allowContentAccess = false + setSupportMultipleWindows(false) + javaScriptCanOpenWindowsAutomatically = false + mediaPlaybackRequiresUserGesture = true + domStorageEnabled = false + setGeolocationEnabled(false) + cacheMode = WebSettings.LOAD_NO_CACHE + } + // The page paints its own themed background; transparency avoids a white flash in dark mode. + setBackgroundColor(android.graphics.Color.TRANSPARENT) + webViewClient = AidenVisualWebViewClient(onRendererGone) + webChromeClient = AidenVisualWebChromeClient() + addJavascriptInterface(bridge, AIDEN_VISUAL_BRIDGE_NAME) +} + +/** + * The one object the page can reach. JavaScript interface calls arrive on a WebView + * background thread, so parsed messages hop to the main thread, and nothing is + * delivered once the viewer has gone. + */ +private class AidenVisualBridgeInterface( + private val onMessage: (AidenVisualBridgeMessage) -> Unit +) { + private val mainHandler = Handler(Looper.getMainLooper()) + @Volatile private var open = true + + fun close() { + open = false + mainHandler.removeCallbacksAndMessages(null) + } + + @JavascriptInterface + fun postMessage(json: String?) { + if (!open || json == null) return + val message = AidenVisualBridgeMessage.parse(json) ?: return + mainHandler.post { if (open) onMessage(message) } + } +} + +/** Denies every navigation after the initial document and every subresource fetch. */ +private class AidenVisualWebViewClient( + private val onRendererGone: () -> Unit +) : WebViewClient() { + override fun shouldOverrideUrlLoading(view: WebView?, request: WebResourceRequest?): Boolean = true + + override fun shouldInterceptRequest(view: WebView?, request: WebResourceRequest?): WebResourceResponse? { + val scheme = request?.url?.scheme?.lowercase() ?: return null + if (scheme !in BLOCKED_SCHEMES) return null + return WebResourceResponse( + "text/plain", + "utf-8", + 403, + "Forbidden", + emptyMap(), + ByteArrayInputStream(ByteArray(0)) + ) + } + + override fun onRenderProcessGone(view: WebView?, detail: RenderProcessGoneDetail?): Boolean { + // The view is unusable now; the screen replaces it, which destroys this one. + onRendererGone() + return true + } + + private companion object { + val BLOCKED_SCHEMES = setOf("http", "https", "file", "content") + } +} + +/** Denies permissions, windows, file pickers and JavaScript dialogs. */ +private class AidenVisualWebChromeClient : WebChromeClient() { + override fun onPermissionRequest(request: PermissionRequest?) { + request?.deny() + } + + override fun onGeolocationPermissionsShowPrompt(origin: String?, callback: GeolocationPermissions.Callback?) { + callback?.invoke(origin, false, false) + } + + override fun onCreateWindow(view: WebView?, isDialog: Boolean, isUserGesture: Boolean, resultMsg: Message?): Boolean = false + + override fun onShowFileChooser( + webView: WebView?, + filePathCallback: ValueCallback>?, + fileChooserParams: WebChromeClient.FileChooserParams? + ): Boolean { + filePathCallback?.onReceiveValue(null) + return true + } + + override fun onJsAlert(view: WebView?, url: String?, message: String?, result: JsResult?): Boolean { + result?.cancel() + return true + } + + override fun onJsConfirm(view: WebView?, url: String?, message: String?, result: JsResult?): Boolean { + result?.cancel() + return true + } + + override fun onJsPrompt( + view: WebView?, + url: String?, + message: String?, + defaultValue: String?, + result: JsPromptResult? + ): Boolean { + result?.cancel() + return true + } +} + +private fun openAidenVisualLink(context: Context, url: String) { + // Checked again here: only a confirmed https link without user info leaves the app. + if (!AidenVisualBridgeMessage.isOpenableUrl(url)) return + try { + context.startActivity( + Intent(Intent.ACTION_VIEW, Uri.parse(url)).addCategory(Intent.CATEGORY_BROWSABLE) + ) + } catch (_: Exception) { + // No browser; the link stays unopened. + } +} + +/** One cache per process over the app's caches directory. */ +private object AidenVisualDocumentCacheStore { + @Volatile private var cache: AidenVisualDocumentCache? = null + + fun get(context: Context): AidenVisualDocumentCache = cache ?: synchronized(this) { + cache ?: AidenVisualDocumentCache(File(context.applicationContext.cacheDir, "visual_documents")) + .also { cache = it } + } +} diff --git a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/models/AidenInstallation.kt b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/models/AidenInstallation.kt index a9f8ec42e..2067e4f56 100644 --- a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/models/AidenInstallation.kt +++ b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/models/AidenInstallation.kt @@ -162,6 +162,10 @@ data class AidenServer( val supportsChatVisuals: Boolean get() = features.contains(AidenRemoteProtocol.CHAT_VISUALS_FEATURE) + /** A tapped visual snapshot opens the Mac's interactive viewer document (revision 34). */ + val supportsChatVisualViewer: Boolean + get() = features.contains(AidenRemoteProtocol.CHAT_VISUAL_VIEWER_FEATURE) + companion object { private val FEATURE_TOKEN = Regex("^[a-z0-9][a-z0-9-]{0,63}$") } diff --git a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/models/AidenVisualDocument.kt b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/models/AidenVisualDocument.kt new file mode 100644 index 000000000..2711f43c7 --- /dev/null +++ b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/models/AidenVisualDocument.kt @@ -0,0 +1,52 @@ +package sbtbiswas.AidenOnTheGo.models + +import sbtbiswas.AidenOnTheGo.config.AidenThemePresetID +import sbtbiswas.AidenOnTheGo.protocol.AidenRemoteProtocol + +/** The light or dark scheme a visual viewer document is drawn in (contract revision 34). */ +enum class AidenVisualDocumentScheme(val wire: String) { + LIGHT("light"), + DARK("dark"); + + companion object { + fun forDarkTheme(isDark: Boolean): AidenVisualDocumentScheme = if (isDark) DARK else LIGHT + } +} + +/** + * Request bounds for `GET /chats/{chatId}/visuals/{visualId}/document` (revision 34). + * Ids are sent as plain path segments, so only the unreserved characters ids carry + * on the wire are accepted, and dot segments are refused so a URL never resolves + * to another route. + */ +object AidenVisualDocumentRequest { + /** Uncompressed body bound the Mac guarantees. */ + const val MAXIMUM_BYTES = 12 * 1_024 * 1_024 + /** The preset the Mac falls back to, and the one the dynamic (Material You) preset maps to. */ + const val DEFAULT_THEME = "aiden" + + private val IDENTIFIER = Regex("^[A-Za-z0-9._:-]{1,256}$") + private val THEME = Regex("^[a-z0-9][a-z0-9-]{0,63}$") + + fun isValidIdentifier(value: String): Boolean = + IDENTIFIER.matches(value) && value != "." && value != ".." + + fun isValidTheme(value: String): Boolean = THEME.matches(value) + + /** + * The wire preset id from `protocol/aiden-appearance-v1.json`. Android's dynamic + * preset has no shared palette, so it asks for Aiden's. + */ + fun themeId(preset: AidenThemePresetID): String = + if (preset.isDynamic) DEFAULT_THEME else preset.name.lowercase() +} + +/** + * Whether tapping a visual snapshot opens the interactive viewer. The live `/server` + * features decide once they are known; before that, the features this pairing last + * saw stand in (R21), so a cold offline start can still show a cached document. + */ +object AidenVisualViewerGate { + fun isAvailable(liveFeatures: List?, cachedFeatures: List?): Boolean = + (liveFeatures ?: cachedFeatures)?.contains(AidenRemoteProtocol.CHAT_VISUAL_VIEWER_FEATURE) == true +} diff --git a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/networking/AidenRemoteClient.kt b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/networking/AidenRemoteClient.kt index 9c912d09f..ccd0488ae 100644 --- a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/networking/AidenRemoteClient.kt +++ b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/networking/AidenRemoteClient.kt @@ -1028,6 +1028,86 @@ class AidenRemoteClient( throw error } + /** + * The self-contained viewer page for one inline visual (contract revision 34, + * `chat-visual-viewer-v1`), drawn in [scheme] and the [theme] preset. The body + * is `text/html; charset=utf-8`, at most [AidenVisualDocumentRequest.MAXIMUM_BYTES] + * once OkHttp has transparently undone any gzip encoding. + */ + suspend fun visualDocument( + chatId: String, + visualId: String, + scheme: AidenVisualDocumentScheme, + theme: String + ): String = try { + if (!AidenVisualDocumentRequest.isValidIdentifier(chatId) || + !AidenVisualDocumentRequest.isValidIdentifier(visualId) || + !AidenVisualDocumentRequest.isValidTheme(theme) + ) { + throw AidenRemoteClientException.InvalidResponse("Invalid visual document request.") + } + withContext(Dispatchers.IO) { + if (credential.isNullOrEmpty()) { + AidenDiagnostics.record(AidenDiagnosticArea.AUTHENTICATION, AidenDiagnosticEvent.REQUEST_FAILED, AidenDiagnosticOutcome.FAILED, AidenDiagnosticCode.UNAUTHORIZED) + throw AidenRemoteClientException.MissingCredential + } + // Ids are limited to unreserved path characters above; the query values are fixed tokens. + val url = "$endpoint/chats/$chatId/visuals/$visualId/document?scheme=${scheme.wire}&theme=$theme" + val request = Request.Builder() + .url(url) + .addHeader("Aiden-Protocol-Version", "1") + .addHeader("Accept", "text/html") + .addHeader("Authorization", "Bearer $credential") + .get() + .build() + + val documentResponse = try { + httpClient.newCall(request).await() + } catch (error: CancellationException) { + throw error + } catch (error: Exception) { + AidenDiagnostics.record(AidenDiagnosticArea.CONNECTION, AidenDiagnosticEvent.REQUEST_FAILED, AidenDiagnosticOutcome.FAILED, AidenDiagnosticCode.NETWORK) + throw error + } + documentResponse.use { response -> + if (!response.isSuccessful) { + val bytes = try { response.body.readBounded(1_048_576) } catch (_: Exception) { ByteArray(0) } + AidenDiagnostics.record( + if (response.code == 401 || response.code == 403) AidenDiagnosticArea.AUTHENTICATION else AidenDiagnosticArea.CONNECTION, + AidenDiagnosticEvent.REQUEST_FAILED, + AidenDiagnosticOutcome.FAILED, + if (response.code == 401 || response.code == 403) AidenDiagnosticCode.UNAUTHORIZED else AidenDiagnosticCode.NETWORK + ) + throw AidenRemoteClientException.Server(response.code, parseError(response.code, bytes)) + } + val contentType = response.header("Content-Type")?.split(";")?.firstOrNull()?.trim()?.lowercase() + if (contentType != "text/html") { + AidenDiagnostics.record(AidenDiagnosticArea.CONTRACT, AidenDiagnosticEvent.CONTRACT_REJECTED, AidenDiagnosticOutcome.FAILED, AidenDiagnosticCode.INVALID_RESPONSE) + throw AidenRemoteClientException.InvalidResponse() + } + val bytes = try { + response.body.readBounded(AidenVisualDocumentRequest.MAXIMUM_BYTES) + } catch (error: CancellationException) { + throw error + } catch (_: ResponseBodyLimitExceededException) { + AidenDiagnostics.record(AidenDiagnosticArea.CONTRACT, AidenDiagnosticEvent.CONTRACT_REJECTED, AidenDiagnosticOutcome.FAILED, AidenDiagnosticCode.INVALID_RESPONSE) + throw AidenRemoteClientException.InvalidResponse() + } catch (error: IOException) { + AidenDiagnostics.record(AidenDiagnosticArea.CONNECTION, AidenDiagnosticEvent.REQUEST_FAILED, AidenDiagnosticOutcome.FAILED, AidenDiagnosticCode.NETWORK) + throw error + } + if (bytes.isEmpty()) { + AidenDiagnostics.record(AidenDiagnosticArea.CONTRACT, AidenDiagnosticEvent.CONTRACT_REJECTED, AidenDiagnosticOutcome.FAILED, AidenDiagnosticCode.INVALID_RESPONSE) + throw AidenRemoteClientException.InvalidResponse() + } + String(bytes, Charsets.UTF_8) + } + } + } catch (error: CancellationException) { + AidenDiagnostics.record(AidenDiagnosticArea.CONNECTION, AidenDiagnosticEvent.REQUEST_FAILED, AidenDiagnosticOutcome.CANCELLED, AidenDiagnosticCode.NETWORK) + throw error + } + suspend fun startTurn( chatId: String, request: AidenTurnStart, diff --git a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/persistence/AidenVisualDocumentCache.kt b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/persistence/AidenVisualDocumentCache.kt new file mode 100644 index 000000000..6b0f8c1d0 --- /dev/null +++ b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/persistence/AidenVisualDocumentCache.kt @@ -0,0 +1,167 @@ +package sbtbiswas.AidenOnTheGo.persistence + +import java.io.File +import java.security.MessageDigest + +/** Identifies one cached viewer document: the pairing, chat, visual, scheme and theme. */ +data class AidenVisualDocumentKey( + /** The paired Mac and this phone's pairing, so a re-pair never reads another pairing's copy. */ + val hostIdentity: String, + val chatId: String, + val visualId: String, + val scheme: String, + val theme: String +) { + /** SHA-256 hex of the fields, NUL-separated so no two keys can run together. */ + val fileName: String + get() = MessageDigest.getInstance("SHA-256") + .digest(listOf(hostIdentity, chatId, visualId, scheme, theme).joinToString("\u0000").toByteArray(Charsets.UTF_8)) + .joinToString("") { "%02x".format(it) } +} + +/** + * The last viewer document per [AidenVisualDocumentKey], shown when the Mac is + * unreachable (contract revision 34, "Offline"). Plain JVM code over [directory] + * (the app's caches directory in production). At most [maximumEntries] files and + * [maximumTotalBytes] in all are kept; the least recently used go first. Reads + * and writes mark a file used. Cache failures never block the live document. + */ +class AidenVisualDocumentCache( + private val directory: File, + private val maximumEntries: Int = 20, + private val maximumTotalBytes: Long = 40L * 1_024 * 1_024, + private val maximumEntryBytes: Int = 12 * 1_024 * 1_024, + private val clock: () -> Long = System::currentTimeMillis +) { + private var lastStamp = 0L + private var stampInitialized = false + + init { + require(maximumEntries > 0) { "Entry limit must be positive" } + require(maximumTotalBytes > 0) { "Size limit must be positive" } + require(maximumEntryBytes > 0) { "Entry size limit must be positive" } + } + + @Synchronized + fun load(key: AidenVisualDocumentKey): String? = try { + val file = file(key) + if (!file.isFile || file.length() !in 1..maximumEntryBytes.toLong()) { + null + } else { + val html = file.readBytes().toString(Charsets.UTF_8) + file.setLastModified(nextStamp()) + html + } + } catch (_: Exception) { + null + } + + /** Returns false when the document was too large to keep or the write failed. */ + @Synchronized + fun store(key: AidenVisualDocumentKey, html: String): Boolean = try { + val target = file(key) + val bytes = html.toByteArray(Charsets.UTF_8) + if (bytes.isEmpty() || bytes.size > maximumEntryBytes || bytes.size > maximumTotalBytes) { + // A stale copy must not outlive a newer document that could not be kept. + target.delete() + false + } else { + directory.mkdirs() + val temporary = File(directory, ".${target.name}.tmp") + temporary.writeBytes(bytes) + if (target.exists()) target.delete() + if (!temporary.renameTo(target)) { + temporary.delete() + false + } else { + target.setLastModified(nextStamp()) + evictOverflow() + true + } + } + } catch (_: Exception) { + false + } + + @Synchronized + fun remove(key: AidenVisualDocumentKey) { + file(key).delete() + } + + private fun evictOverflow() { + val entries = directory.listFiles { file -> file.isFile && file.extension == EXTENSION } + ?.sortedWith(compareByDescending { it.lastModified() }.thenByDescending { it.name }) + ?: return + var count = 0 + var total = 0L + for (entry in entries) { + val size = entry.length() + if (count < maximumEntries && total + size <= maximumTotalBytes) { + count++ + total += size + } else { + entry.delete() + } + } + } + + // A strictly increasing file time orders eviction by use. Steps are a whole second + // so file systems with coarse modification times keep the order, and the first + // stamp after launch continues from the newest file already on disk. + private fun nextStamp(): Long { + if (!stampInitialized) { + stampInitialized = true + val newest = directory.listFiles { file -> file.isFile && file.extension == EXTENSION } + ?.maxOfOrNull { it.lastModified() } ?: 0L + lastStamp = maxOf(lastStamp, newest) + } + lastStamp = maxOf(clock(), lastStamp + 1_000) + return lastStamp + } + + private fun file(key: AidenVisualDocumentKey): File = File(directory, "${key.fileName}.$EXTENSION") + + private companion object { + const val EXTENSION = "html" + } +} + +/** What the viewer shows after one load attempt. */ +sealed interface AidenVisualDocumentLoad { + /** The Mac's current document, now also cached. */ + data class Fresh(val html: String) : AidenVisualDocumentLoad + + /** The Mac could not be reached; the last cached copy, which may be out of date. */ + data class Cached(val html: String) : AidenVisualDocumentLoad + + /** Nothing to show. [unavailable] is the Mac's `not_found`: the visual or the viewer is gone. */ + data class Failed(val unavailable: Boolean) : AidenVisualDocumentLoad +} + +/** + * Fetches a viewer document and keeps the cache in step: a fresh document replaces + * the cached copy; a failed fetch falls back to it; a `404` drops it, since the + * chat or visual no longer exists on the Mac. + */ +class AidenVisualDocumentLoader( + private val cache: AidenVisualDocumentCache, + private val fetch: suspend () -> String, + private val isNotFound: (Throwable) -> Boolean +) { + suspend fun load(key: AidenVisualDocumentKey): AidenVisualDocumentLoad { + val html = try { + fetch() + } catch (error: kotlinx.coroutines.CancellationException) { + throw error + } catch (error: Exception) { + if (isNotFound(error)) { + cache.remove(key) + return AidenVisualDocumentLoad.Failed(unavailable = true) + } + return cache.load(key)?.let { AidenVisualDocumentLoad.Cached(it) } + ?: AidenVisualDocumentLoad.Failed(unavailable = false) + } + cache.store(key, html) + return AidenVisualDocumentLoad.Fresh(html) + } +} diff --git a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/protocol/AidenRemoteProtocol.kt b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/protocol/AidenRemoteProtocol.kt index f33f1f27b..520e82328 100644 --- a/android/app/src/main/java/sbtbiswas/AidenOnTheGo/protocol/AidenRemoteProtocol.kt +++ b/android/app/src/main/java/sbtbiswas/AidenOnTheGo/protocol/AidenRemoteProtocol.kt @@ -77,6 +77,9 @@ object AidenRemoteProtocol { /** Contract revision 32: assistant messages may carry native `appPanels`, read and * answered behind the `app-controls:read` and `app-controls:respond` grants. */ const val CHAT_UI_PANELS_FEATURE = "chat-ui-panels-enabled-v1" + /** Contract revision 34: `GET /chats/{chatId}/visuals/{visualId}/document`, the + * self-contained page a phone opens full screen when a visual's snapshot is tapped. */ + const val CHAT_VISUAL_VIEWER_FEATURE = "chat-visual-viewer-v1" const val MAX_QUESTION_COUNT = 4 const val MIN_QUESTION_OPTIONS = 2 const val MAX_QUESTION_OPTIONS = 5 diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml index ca7372fd2..fb927ccaf 100644 --- a/android/app/src/main/res/values/strings.xml +++ b/android/app/src/main/res/values/strings.xml @@ -857,6 +857,23 @@ Show more Show less + + Open interactive + Close visual + Show snapshot image + Loading visual + Showing a saved copy. May be out of date. + Couldn’t load this visual. Check that your Mac is reachable. + This visual isn’t available on your Mac. + The visual stopped responding. + Add to message? + The visual suggests this follow-up. It’s added to your message so you can edit it before sending. + “%1$s” + Add to message + Dismiss + Open link? + Open + Message actions Select text diff --git a/android/app/src/test/java/sbtbiswas/AidenOnTheGo/AidenVisualDocumentClientTest.kt b/android/app/src/test/java/sbtbiswas/AidenOnTheGo/AidenVisualDocumentClientTest.kt new file mode 100644 index 000000000..d222b2164 --- /dev/null +++ b/android/app/src/test/java/sbtbiswas/AidenOnTheGo/AidenVisualDocumentClientTest.kt @@ -0,0 +1,205 @@ +package sbtbiswas.AidenOnTheGo + +import kotlinx.coroutines.runBlocking +import okhttp3.OkHttpClient +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import okio.Buffer +import okio.GzipSink +import okio.buffer +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Assert.fail +import org.junit.Before +import org.junit.Test +import sbtbiswas.AidenOnTheGo.config.AidenThemePresetID +import sbtbiswas.AidenOnTheGo.models.AidenInstallation +import sbtbiswas.AidenOnTheGo.models.AidenServer +import sbtbiswas.AidenOnTheGo.models.AidenVisualDocumentRequest +import sbtbiswas.AidenOnTheGo.models.AidenVisualDocumentScheme +import sbtbiswas.AidenOnTheGo.models.AidenVisualViewerGate +import sbtbiswas.AidenOnTheGo.networking.AidenRemoteClient +import sbtbiswas.AidenOnTheGo.protocol.AidenRemoteCapability +import sbtbiswas.AidenOnTheGo.protocol.AidenRemoteClientException +import sbtbiswas.AidenOnTheGo.protocol.AidenRemoteErrorCode +import java.time.Instant +import java.util.concurrent.TimeUnit + +/** `GET /chats/{chatId}/visuals/{visualId}/document` and its feature gate (contract revision 34). */ +class AidenVisualDocumentClientTest { + private lateinit var server: MockWebServer + private lateinit var client: AidenRemoteClient + + @Before + fun setup() { + server = MockWebServer() + server.start() + val installation = AidenInstallation( + instanceId = "test_instance", + deviceId = "test_device", + name = "Test Mac", + endpoint = server.url("/api/aiden/v1").toString(), + serverSpkiSha256 = "sha256/test", + deviceCapabilities = listOf(AidenRemoteCapability.CHAT_READ), + serverCapabilities = listOf(AidenRemoteCapability.CHAT_READ), + createdAt = Instant.now() + ) + client = AidenRemoteClient( + installation = installation, + credential = "test_credential_123", + customOkHttpClient = OkHttpClient.Builder().build() + ) + } + + @After + fun teardown() { + server.shutdown() + } + + @Test + fun requestsTheDocumentForTheSchemeAndThemeWithChatReadAuthentication() = runBlocking { + val html = "

Résumé chart ✓

" + server.enqueue( + MockResponse() + .setResponseCode(200) + .setHeader("Content-Type", "text/html; charset=utf-8") + .setHeader("ETag", "\"doc-1\"") + .setBody(html) + ) + + val document = client.visualDocument("chat-1", "visual_1", AidenVisualDocumentScheme.DARK, "dusk") + val request = server.takeRequest(5, TimeUnit.SECONDS)!! + + assertEquals(html, document) + assertEquals("GET", request.method) + assertEquals("/api/aiden/v1/chats/chat-1/visuals/visual_1/document", request.requestUrl!!.encodedPath) + assertEquals("dark", request.requestUrl!!.queryParameter("scheme")) + assertEquals("dusk", request.requestUrl!!.queryParameter("theme")) + assertEquals(setOf("scheme", "theme"), request.requestUrl!!.queryParameterNames) + assertEquals("Bearer test_credential_123", request.getHeader("Authorization")) + assertEquals("1", request.getHeader("Aiden-Protocol-Version")) + assertTrue(request.getHeader("Accept")!!.contains("text/html")) + } + + @Test + fun acceptsAndDecodesAGzippedDocument() = runBlocking { + val html = "" + "
row
".repeat(2_000) + "" + val gzipped = Buffer() + GzipSink(gzipped).buffer().use { it.writeUtf8(html) } + server.enqueue( + MockResponse() + .setResponseCode(200) + .setHeader("Content-Type", "text/html; charset=utf-8") + .setHeader("Content-Encoding", "gzip") + .setBody(gzipped) + ) + + val document = client.visualDocument("chat-1", "visual-1", AidenVisualDocumentScheme.LIGHT, "aiden") + val request = server.takeRequest(5, TimeUnit.SECONDS)!! + + assertEquals(html, document) + assertEquals("gzip", request.getHeader("Accept-Encoding")) + assertEquals("light", request.requestUrl!!.queryParameter("scheme")) + } + + @Test + fun mapsNotFoundAndInvalidRequestToTheMacsErrorEnvelope() = runBlocking { + server.enqueue( + MockResponse() + .setResponseCode(404) + .setHeader("Content-Type", "application/json") + .setBody("""{"error":{"code":"not_found","message":"Visual not found.","requestId":"req_1","retryable":false}}""") + ) + try { + client.visualDocument("chat-1", "gone", AidenVisualDocumentScheme.LIGHT, "aiden") + fail("A 404 must not decode as a document") + } catch (error: AidenRemoteClientException.Server) { + assertEquals(404, error.statusCode) + assertEquals(AidenRemoteErrorCode.NOT_FOUND, error.body.code) + assertFalse(error.isCredentialRevoked) + } + + server.enqueue( + MockResponse() + .setResponseCode(400) + .setHeader("Content-Type", "application/json") + .setBody("""{"error":{"code":"invalid_request","message":"Bad theme.","requestId":"req_2","retryable":false}}""") + ) + try { + client.visualDocument("chat-1", "visual-1", AidenVisualDocumentScheme.LIGHT, "aiden") + fail("A 400 must not decode as a document") + } catch (error: AidenRemoteClientException.Server) { + assertEquals(400, error.statusCode) + assertEquals(AidenRemoteErrorCode.INVALID_REQUEST, error.body.code) + } + } + + @Test + fun rejectsANonHtmlBodyAndOneOverTheBound() { + server.enqueue( + MockResponse().setResponseCode(200).setHeader("Content-Type", "application/json").setBody("{}") + ) + assertThrows(AidenRemoteClientException.InvalidResponse::class.java) { + runBlocking { client.visualDocument("chat-1", "visual-1", AidenVisualDocumentScheme.LIGHT, "aiden") } + } + + server.enqueue( + MockResponse() + .setResponseCode(200) + .setHeader("Content-Type", "text/html; charset=utf-8") + .setBody("x".repeat(AidenVisualDocumentRequest.MAXIMUM_BYTES + 1)) + ) + assertThrows(AidenRemoteClientException.InvalidResponse::class.java) { + runBlocking { client.visualDocument("chat-1", "visual-1", AidenVisualDocumentScheme.LIGHT, "aiden") } + } + } + + @Test + fun refusesIdsAndThemesThatWouldLeaveTheRouteWithoutSendingAnything() { + val refused = listOf( + Triple("chat/1", "visual-1", "aiden"), + Triple("..", "visual-1", "aiden"), + Triple("chat-1", ".", "aiden"), + Triple("chat-1", "visual?x=1", "aiden"), + Triple("chat-1", "", "aiden"), + Triple("chat-1", "visual-1", "Dusk"), + Triple("chat-1", "visual-1", "aiden&scheme=dark") + ) + for ((chatId, visualId, theme) in refused) { + assertThrows("$chatId $visualId $theme", AidenRemoteClientException.InvalidResponse::class.java) { + runBlocking { client.visualDocument(chatId, visualId, AidenVisualDocumentScheme.LIGHT, theme) } + } + } + assertEquals(0, server.requestCount) + } + + @Test + fun theThemeIsTheSharedPresetIdAndTheDynamicPresetAsksForAiden() { + assertEquals("aiden", AidenVisualDocumentRequest.themeId(AidenThemePresetID.AIDEN)) + assertEquals("monochrome", AidenVisualDocumentRequest.themeId(AidenThemePresetID.MONOCHROME)) + assertEquals("aiden", AidenVisualDocumentRequest.themeId(AidenThemePresetID.DYNAMIC)) + for (preset in AidenThemePresetID.entries) { + assertTrue(AidenVisualDocumentRequest.isValidTheme(AidenVisualDocumentRequest.themeId(preset))) + } + } + + @Test + fun theViewerOpensOnlyWhenTheMacAnnouncedTheToken() { + val token = "chat-visual-viewer-v1" + assertFalse(AidenVisualViewerGate.isAvailable(listOf("chat-visuals-v1"), null)) + assertTrue(AidenVisualViewerGate.isAvailable(listOf("chat-visuals-v1", token), null)) + // Before /server answers, the features this pairing last saw stand in. + assertTrue(AidenVisualViewerGate.isAvailable(null, listOf(token))) + assertFalse(AidenVisualViewerGate.isAvailable(null, null)) + // A live answer without the token wins over an older cached one. + assertFalse(AidenVisualViewerGate.isAvailable(listOf("chat-visuals-v1"), listOf(token))) + + val withToken = AidenServer(instanceId = "mac", name = "Mac", capabilities = emptyList(), features = listOf(token)) + val without = AidenServer(instanceId = "mac", name = "Mac", capabilities = emptyList(), features = listOf("chat-visuals-v1")) + assertTrue(withToken.supportsChatVisualViewer) + assertFalse(without.supportsChatVisualViewer) + } +} diff --git a/android/app/src/test/java/sbtbiswas/AidenOnTheGo/features/chat/AidenVisualViewerModelTest.kt b/android/app/src/test/java/sbtbiswas/AidenOnTheGo/features/chat/AidenVisualViewerModelTest.kt new file mode 100644 index 000000000..a127632a2 --- /dev/null +++ b/android/app/src/test/java/sbtbiswas/AidenOnTheGo/features/chat/AidenVisualViewerModelTest.kt @@ -0,0 +1,108 @@ +package sbtbiswas.AidenOnTheGo.features.chat + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +/** The viewer page's message channel (contract revision 34) and what a confirmed prompt does. */ +class AidenVisualViewerModelTest { + @Test + fun parsesTheClosedMessageSet() { + assertEquals(AidenVisualBridgeMessage.Ready, AidenVisualBridgeMessage.parse("""{"type":"ready"}""")) + assertEquals( + AidenVisualBridgeMessage.Prompt("Compare Q3 with Q2"), + AidenVisualBridgeMessage.parse("""{"type":"prompt","text":" Compare Q3 with Q2\n"}""") + ) + assertEquals( + AidenVisualBridgeMessage.OpenUrl("https://example.com/report?q=1#top"), + AidenVisualBridgeMessage.parse("""{"type":"openUrl","url":"https://example.com/report?q=1#top"}""") + ) + // Extra keys on a known message do not reject it. + assertEquals(AidenVisualBridgeMessage.Ready, AidenVisualBridgeMessage.parse("""{"type":"ready","at":12}""")) + } + + @Test + fun ignoresUnknownTypesAndMalformedMessages() { + val ignored = listOf( + """{"type":"sendPrompt","text":"hi"}""", + """{"type":"resize","height":300}""", + """{"type":"READY"}""", + """{"type":1}""", + """{"text":"no type"}""", + """["ready"]""", + """"ready"""", + "ready", + "", + "{", + "[object Object]", + "null" + ) + for (raw in ignored) assertNull(raw, AidenVisualBridgeMessage.parse(raw)) + } + + @Test + fun ignoresPromptsThatAreNotBoundedText() { + val limit = AidenVisualBridgeMessage.MAXIMUM_PROMPT_LENGTH + assertNull(AidenVisualBridgeMessage.parse("""{"type":"prompt","text":42}""")) + assertNull(AidenVisualBridgeMessage.parse("""{"type":"prompt","text":["a"]}""")) + assertNull(AidenVisualBridgeMessage.parse("""{"type":"prompt","text":{"value":"a"}}""")) + assertNull(AidenVisualBridgeMessage.parse("""{"type":"prompt","text":null}""")) + assertNull(AidenVisualBridgeMessage.parse("""{"type":"prompt"}""")) + assertNull(AidenVisualBridgeMessage.parse("""{"type":"prompt","text":" "}""")) + assertNull(AidenVisualBridgeMessage.parse("""{"type":"prompt","text":"${"a".repeat(limit + 1)}"}""")) + assertEquals( + AidenVisualBridgeMessage.Prompt("a".repeat(limit)), + AidenVisualBridgeMessage.parse("""{"type":"prompt","text":"${"a".repeat(limit)}"}""") + ) + } + + @Test + fun opensOnlyHttpsLinksWithAHostAndNoUserInfo() { + val refused = listOf( + "http://example.com", + "javascript:alert(1)", + "file:///etc/hosts", + "content://sbtbiswas.AidenOnTheGo/secret", + "intent://scan#Intent;scheme=zxing;end", + "data:text/html,hi", + "https://user:pass@example.com/", + "https://user@example.com/", + "https:///path-only", + "https:example.com", + "//example.com", + "https://exa mple.com", + "https://example.com/\nnext", + "https://example.com/" + "a".repeat(AidenVisualBridgeMessage.MAXIMUM_URL_LENGTH) + ) + for (url in refused) { + assertFalse(url, AidenVisualBridgeMessage.isOpenableUrl(url)) + val escaped = url.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n") + assertNull(url, AidenVisualBridgeMessage.parse("""{"type":"openUrl","url":"$escaped"}""")) + } + assertNull(AidenVisualBridgeMessage.parse("""{"type":"openUrl","url":7}""")) + assertTrue(AidenVisualBridgeMessage.isOpenableUrl("https://example.com")) + assertTrue(AidenVisualBridgeMessage.isOpenableUrl("https://sub.example.com:8443/a/b?c=d")) + } + + @Test + fun aConfirmedPromptIsAppendedToTheDraftNeverReplacingIt() { + assertEquals("Show March", AidenVisualPromptDraft.appended("", " Show March ")) + assertEquals("Draft so far\n\nShow March", AidenVisualPromptDraft.appended("Draft so far \n", "Show March")) + assertNull(AidenVisualPromptDraft.appended("Draft so far", " ")) + } + + @Test + fun promptsAndLinksAreLimitedToOneEveryThreeSeconds() { + var now = 10_000L + val throttle = AidenVisualBridgeThrottle { now } + assertTrue(throttle.tryAcquire()) + now += 2_999 + assertFalse(throttle.tryAcquire()) + now += 1 + assertTrue(throttle.tryAcquire()) + now += 3_000 + assertTrue(throttle.tryAcquire()) + } +} diff --git a/android/app/src/test/java/sbtbiswas/AidenOnTheGo/persistence/AidenVisualDocumentCacheTest.kt b/android/app/src/test/java/sbtbiswas/AidenOnTheGo/persistence/AidenVisualDocumentCacheTest.kt new file mode 100644 index 000000000..8d5e3fa49 --- /dev/null +++ b/android/app/src/test/java/sbtbiswas/AidenOnTheGo/persistence/AidenVisualDocumentCacheTest.kt @@ -0,0 +1,149 @@ +package sbtbiswas.AidenOnTheGo.persistence + +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import sbtbiswas.AidenOnTheGo.protocol.AidenRemoteClientException +import sbtbiswas.AidenOnTheGo.protocol.AidenRemoteErrorCode +import sbtbiswas.AidenOnTheGo.protocol.AidenRemoteErrorEnvelope +import java.io.IOException + +class AidenVisualDocumentCacheTest { + @get:Rule + val tempFolder = TemporaryFolder() + + private var now = 1_700_000_000_000L + private val clock = { now } + + private fun key(visualId: String, scheme: String = "dark", theme: String = "dusk", host: String = "mac-1\u001fphone-1") = + AidenVisualDocumentKey(host, "chat-1", visualId, scheme, theme) + + private fun cache(entries: Int = 20, totalBytes: Long = 40L * 1_024 * 1_024, entryBytes: Int = 12 * 1_024 * 1_024) = + AidenVisualDocumentCache(tempFolder.root, entries, totalBytes, entryBytes, clock) + + @Test + fun theFileNameIsTheSha256OfEveryKeyField() { + // Independent oracle: SHA-256 of "mac-1\u001fphone-1\0chat-1\0visual-1\0dark\0dusk". + assertEquals( + "c40ef22cb5e804dd2b5a43b708409c15dba0da9febda6c821e32b0741be2a4ef", + key("visual-1").fileName + ) + val variants = listOf( + key("visual-1", scheme = "light"), + key("visual-1", theme = "aiden"), + key("visual-2"), + key("visual-1", host = "mac-1\u001fphone-2"), + AidenVisualDocumentKey("mac-1\u001fphone-1", "chat-2", "visual-1", "dark", "dusk"), + // Field boundaries cannot be shifted to collide. + AidenVisualDocumentKey("mac-1\u001fphone-1", "chat-1visual-1", "", "dark", "dusk") + ) + assertEquals(variants.size + 1, (variants.map { it.fileName } + key("visual-1").fileName).toSet().size) + } + + @Test + fun aDocumentSurvivesANewCacheInstanceAndIsKeyedBySchemeAndTheme() { + cache().store(key("visual-1"), "dark dusk ✓") + + val reopened = cache() + assertEquals("dark dusk ✓", reopened.load(key("visual-1"))) + assertNull(reopened.load(key("visual-1", scheme = "light"))) + assertNull(reopened.load(key("visual-1", theme = "aiden"))) + assertNull(reopened.load(key("visual-1", host = "mac-2\u001fphone-1"))) + } + + @Test + fun evictsTheLeastRecentlyUsedBeyondTheEntryLimit() { + val cache = cache(entries = 3) + for (id in listOf("a", "b", "c")) { + cache.store(key(id), "$id") + now += 10 + } + // Reading "a" makes "b" the least recently used. + assertEquals("a", cache.load(key("a"))) + now += 10 + cache.store(key("d"), "d") + + assertNull(cache.load(key("b"))) + assertEquals("a", cache.load(key("a"))) + assertEquals("c", cache.load(key("c"))) + assertEquals("d", cache.load(key("d"))) + assertEquals(3, tempFolder.root.listFiles { file -> file.extension == "html" }!!.size) + } + + @Test + fun evictsTheLeastRecentlyUsedBeyondTheByteLimit() { + val cache = cache(totalBytes = 10) + cache.store(key("a"), "aaaa") + cache.store(key("b"), "bbbb") + cache.store(key("c"), "cccc") + + assertNull(cache.load(key("a"))) + assertEquals("bbbb", cache.load(key("b"))) + assertEquals("cccc", cache.load(key("c"))) + } + + @Test + fun recencySurvivesARelaunch() { + cache(entries = 2).store(key("a"), "aaaa") + cache(entries = 2).store(key("b"), "bbbb") + // A later launch whose clock reads earlier than the stamps already on disk. + now -= 60_000 + val relaunched = cache(entries = 2) + assertEquals("aaaa", relaunched.load(key("a"))) + relaunched.store(key("c"), "cccc") + + assertNull(relaunched.load(key("b"))) + assertEquals("aaaa", relaunched.load(key("a"))) + } + + @Test + fun aDocumentTooLargeToKeepDropsTheOlderCopy() { + val cache = cache(entryBytes = 8) + assertTrue(cache.store(key("a"), "small")) + assertFalse(cache.store(key("a"), "much too large")) + assertNull(cache.load(key("a"))) + } + + @Test + fun theLoaderCachesFreshDocumentsAndFallsBackWhenTheMacIsUnreachable() = runBlocking { + val cache = cache() + var answer: () -> String = { "v1" } + val loader = AidenVisualDocumentLoader(cache, fetch = { answer() }, isNotFound = ::isNotFound) + + assertEquals(AidenVisualDocumentLoad.Fresh("v1"), loader.load(key("a"))) + answer = { throw IOException("offline") } + assertEquals(AidenVisualDocumentLoad.Cached("v1"), loader.load(key("a"))) + assertEquals(AidenVisualDocumentLoad.Failed(unavailable = false), loader.load(key("never-seen"))) + + answer = { "v2" } + assertEquals(AidenVisualDocumentLoad.Fresh("v2"), loader.load(key("a"))) + assertEquals("v2", cache.load(key("a"))) + } + + @Test + fun aVisualTheMacNoLongerHasIsDroppedFromTheCache() = runBlocking { + val cache = cache() + cache.store(key("a"), "old") + val loader = AidenVisualDocumentLoader( + cache, + fetch = { + throw AidenRemoteClientException.Server( + 404, + AidenRemoteErrorEnvelope.Body(AidenRemoteErrorCode.NOT_FOUND, "Visual not found.", "req_1", false) + ) + }, + isNotFound = ::isNotFound + ) + + assertEquals(AidenVisualDocumentLoad.Failed(unavailable = true), loader.load(key("a"))) + assertNull(cache.load(key("a"))) + } +} + +private fun isNotFound(error: Throwable): Boolean = + error is AidenRemoteClientException.Server && error.statusCode == 404 diff --git a/docs/aiden-remote-api-v1.md b/docs/aiden-remote-api-v1.md index 1588b3495..224676e20 100644 --- a/docs/aiden-remote-api-v1.md +++ b/docs/aiden-remote-api-v1.md @@ -750,3 +750,12 @@ A host announces `bot-attachments-v1` only while it stages phone uploads for Bot - `POST /bots/{botId}/messages` accepts `attachmentIds` (1–10 unique references). With them the text may be empty. The references are spent once inside the idempotency ledger, so a retried request replays its receipt without spending them again. References are scoped to the uploading device and Bot: another device's send is `handle_wrong_device`, and a Bot reference can never be spent in a workspace chat (or a chat reference in a Bot). Images reach the Bot as image input, exactly as the desktop Bot chat sends them; text files follow the words as `name:` and their contents. A host without the feature refuses `attachmentIds` as `invalid_request` and answers the upload routes with `not_found`. + +## Phone visual viewer, revision 34 (`chat-visual-viewer-v1`, provisional) + +A host announces `chat-visual-viewer-v1` when it ships the viewer bundle (`npm run build:visual-viewer`, part of `npm run build`). Phones keep the inline snapshot and open a visual full screen on tap: + +- `GET /chats/{chatId}/visuals/{visualId}/document?scheme=light|dark&theme=` needs `chat:read` and read access to the chat. `visualId` is any id the chat projection lists in a message's `visuals`; anything else is `not_found`. Both query keys are optional (`light`, `aiden`); a preset id the host does not know falls back to `aiden`, and a malformed value or any other key is `invalid_request`. +- The body is one self-contained `text/html` document of at most 12 MiB: the viewer page, its payload, and for an HTML visual the sandboxed guest with only the host libraries its source names (Chart.js, Plotly, KaTeX) inlined. It loads nothing from the network. Responses are `no-store`, carry a strong ETag (`If-None-Match` gets 304) and are gzip-compressed when the client accepts it. +- Native hosts load it with no base URL, deny every other navigation and window, and turn off file access and persistent storage. The page talks to them only through iOS `window.webkit.messageHandlers.aidenVisual` or Android `window.AidenVisualBridge`, with `{"type":"ready"}`, `{"type":"prompt","text":…}` (at most 2,000 characters, one per 3 s) and `{"type":"openUrl","url":"https://…"}`. A prompt is never sent by itself: the phone shows its own confirmation and only adds the text to the composer. A link opens only after the phone asks. +- Visual entries are unchanged, so older phones keep showing snapshots. Phones may cache a document per host, chat, visual, scheme and theme and show it while the Mac is unreachable. diff --git a/docs/plans/README.md b/docs/plans/README.md index 3f8c6c34f..106a702fd 100644 --- a/docs/plans/README.md +++ b/docs/plans/README.md @@ -9,6 +9,7 @@ Research addition 2026-10-03 against `main` at `3f86d41af` (0.52.0): [all-PR inv ## Active and partial - [Built-in Aiden knowledge and app actions](aiden-app-knowledge-actions-plan.md) — Merged in [PR #312](https://github.com/sambitcreate/aiden-agent/pull/312) (2026-10-15); ships in 0.57.0. Physical release acceptance remains open. Bundled offline help and owned skill, real json-render desktop cards, native iOS/Android paired-host controls and Pi CLI selectors/admin commands. Remote revision 19, explicit owner consent, domain gates, replay receipts and registered behavioral tests. Uses pinned Pi 0.87.1; Pi 1.0 integration and wider actions remain separate follow-ups. [Contract research](aiden-chat-controls-json-render-research.md). +- [Interactive visuals on phones](phone-visual-viewer-plan.md) — In progress (2026-10-16) on `claude/project-thread-jxkc0z`. Owner chose tap to open: phones keep the inline snapshot and open a visual full screen in a locked-down web view served by the Mac (`GET /chats/{chatId}/visuals/{visualId}/document`, Remote revision 34 provisional, `chat-visual-viewer-v1`). Covers `render_ui` and `render_artifact`; follow-ups confirm on the phone and only fill the composer. - [Quick View cards](../superpowers/plans/2026-10-15-quick-view-cards.md) — Merged in [PR #487](https://github.com/sambitcreate/aiden-agent/pull/487) (2026-10-15); ships in 0.57.0. Quick View becomes Claude-Code-style stacked cards (Environment, Changes, Context, Subagents) at `SidebarListItem` density. - **Behaviour:** cards start collapsed and reorder by a right-edge grip. Each has three tiers: a summary, inline detail, and an open-in-sidebar or open-external button. - **Visibility:** cards and rows appear only once relevant. Changes needs work or a PR, the PR row needs a created PR, Context needs the first message, and Subagents needs a run started. diff --git a/docs/plans/phone-visual-viewer-plan.md b/docs/plans/phone-visual-viewer-plan.md new file mode 100644 index 000000000..b2dcd5b18 --- /dev/null +++ b/docs/plans/phone-visual-viewer-plan.md @@ -0,0 +1,52 @@ +# Interactive visuals on phones (tap to open) + +Owner decision 2026-10-16: phones keep the inline snapshot and open a visual full screen in a locked-down web view when it is tapped. This is the "Open interactive" step that the [inline generative UI spec](../superpowers/specs/2026-10-08-inline-generative-ui-design.md) §9 left optional. Live inline web views and native catalog renderers were considered and not chosen (see the thread "Charts and visuals" in the Aiden project). + +## Goal + +A chart or widget in a reply on iOS or Android behaves like it does on the Mac when opened: tooltips, tabs, sliders, zoom and Plotly interactions work, in the phone's own theme and light or dark scheme. Both kinds of visual are covered: + +- `render_ui` (native catalog tree, the model's default) is drawn by the same React `AidenUiBlock` the desktop uses. +- `render_artifact` (sandboxed HTML) runs in the same opaque-origin `sandbox="allow-scripts"` iframe and CSP as desktop, with the host libraries it uses inlined. + +## Contract (Remote revision 34, `chat-visual-viewer-v1`) + +Provisional: re-claim the next revision after `main`'s at merge. + +- **Feature token** `chat-visual-viewer-v1`. Announced only when the host has chats and its viewer bundle is installed (`build/renderer/visual-viewer/`). Clients without it keep today's snapshot behaviour (tap opens the image viewer). +- **No new key on visual entries.** `visuals` entries are parsed with a closed key set on every client, so a new key would drop every visual on older phones. Any visual on a host announcing the token can be opened. +- **`GET /chats/{chatId}/visuals/{visualId}/document`** + - Auth: `chat:read` and current read access to the chat (same as attachment content). + - Query (both optional, nothing else allowed): `scheme=light|dark` (default `light`), `theme=` from `protocol/aiden-appearance-v1.json` (default `aiden`; an id the host does not know falls back to `aiden`, a malformed one is `invalid_request`). + - `visualId` is an id the chat projection lists in some message's `visuals`. + - 200 `text/html; charset=utf-8`, a self-contained document (no subresources), with `content-security-policy` matching its meta policy, `cache-control: no-store`, a strong `etag`, and gzip when the client accepts it. A matching `If-None-Match` gets 304. Bounded at 12 MiB uncompressed. + - 404 `not_found` for an unknown chat, visual, missing HTML or a host without the viewer bundle. 400 `invalid_request` for a bad query. +- **Document behaviour** (what native hosts must provide): + - Load the body as an HTML string with no base URL (`loadHTMLString(_:baseURL: nil)` / `loadDataWithBaseURL(null, …)`), JavaScript on, every other navigation and every window open denied, no file or content access, non-persistent storage, no auto-playing media. + - The page talks to native only through one message channel: iOS `window.webkit.messageHandlers.aidenVisual.postMessage(object)`, Android `window.AidenVisualBridge.postMessage(jsonString)`. Messages (closed set, unknown types ignored): + - `{"type":"ready"}` once the visual has drawn. + - `{"type":"prompt","text":"…"}` when the visual asks for a follow-up (`sendPrompt` in HTML, a `sendPrompt` action in `render_ui`). The page has already bounded it to 2,000 characters and one per 3 s. Native shows its own confirmation (quote the text, **Add to message** and **Dismiss**); **Add to message** closes the viewer and appends the text to the chat composer. It never sends by itself, matching desktop's confirm-only rule. + - `{"type":"openUrl","url":"https://…"}` for a `render_ui` `openUrl` action. Native asks before opening it in the system browser; only `https:` is accepted. +- **Offline.** Phones cache the last document per host, chat, visual, scheme and theme (bounded, app caches directory) and show it when the Mac is unreachable, with a note that it may be out of date. + +## Desktop + +1. `renderer/visual-viewer/main.tsx`: the viewer page. Reads its payload from ``; + if (!include(name)) { + inlined = inlined.replace(tag, ""); + continue; + } const source = name === GENERATIVE_UI_KIT_LIB ? generativeUiKitCss() : libraries[name]; if (!source) { throw new Error(`Export is missing host library ${name}.`); } - if (name.endsWith(".css")) { - guestDocument = guestDocument.replace( - ``, - ``, - ); - } else { - guestDocument = guestDocument.replace( - ``, - ``, - ); - } + // A function replacement, so `$&` and friends in library source stay literal. + inlined = inlined.replace(tag, () => (name.endsWith(".css") ? `` : ``)); } + return inlined; +} + +export function generativeUiExportDocument( + html: string, + title: string, + libraries: Readonly>, + theme?: GenerativeUiThemeTokens, +): string { + const guestDocument = inlineGenerativeUiHostLibraries(wrapGenerativeUiHtml(html, title, theme), libraries); const safeTitle = escapeHtml(title); return ` diff --git a/main/services/visual-viewer-document.test.ts b/main/services/visual-viewer-document.test.ts new file mode 100644 index 000000000..9e54e16f5 --- /dev/null +++ b/main/services/visual-viewer-document.test.ts @@ -0,0 +1,120 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { VISUAL_VIEWER_CSP, VISUAL_VIEWER_PAYLOAD_ELEMENT_ID, type VisualViewerPayload } from "../../renderer/shared/visual-viewer.js"; +import type { ChatUiVisualV1 } from "../../renderer/shared/aiden-ui/types.js"; +import { + MAX_VISUAL_VIEWER_DOCUMENT_BYTES, + buildVisualViewerDocument, + parseVisualViewerQuery, + visualViewerDocumentEtag, +} from "./visual-viewer-document.js"; + +const BUNDLE = { script: "window.__viewer = true;", style: "main{color:red}" }; +const LIBRARIES = { + "chart.js": "/* CHART-SOURCE */ window.Chart = function () {};", + "plotly.js": "/* PLOTLY-SOURCE */ window.Plotly = {};", + "katex.js": "/* KATEX-SOURCE */", + "katex.css": "/* KATEX-STYLE */", +}; +const LIGHT = { scheme: "light" as const, preset: "aiden" as const }; + +function payloadOf(document: string): VisualViewerPayload { + const match = new RegExp(``, "u").exec(document); + assert.ok(match, "the document carries its payload element"); + return JSON.parse(match[1]!) as VisualViewerPayload; +} + +const UI_VISUAL: ChatUiVisualV1 = { + version: 1, + kind: "ui", + id: "ui-1", + title: "Plan options", + catalogVersion: 1, + tree: { t: "Visual", k: "0", c: [{ t: "#text", k: "0.0", s: "Team" }] }, + fallbackText: "Plan options: Team", +}; + +test("viewer query defaults to the light Aiden theme and falls back for presets this host does not know", () => { + assert.deepEqual(parseVisualViewerQuery(new URLSearchParams("")), LIGHT); + assert.deepEqual(parseVisualViewerQuery(new URLSearchParams("scheme=dark&theme=midnight")), { scheme: "dark", preset: "midnight" }); + assert.deepEqual(parseVisualViewerQuery(new URLSearchParams("theme=aurora-2")), LIGHT); + for (const bad of ["scheme=sepia", "theme=Mid night", "theme=%3Cx%3E", "scheme=dark&scheme=light", "theme=aiden&extra=1"]) { + assert.throws(() => parseVisualViewerQuery(new URLSearchParams(bad)), undefined, bad); + } +}); + +test("an HTML visual travels as a sandboxed guest that cannot break out of the payload", () => { + const html = `
\u2028 `; + const document = buildVisualViewerDocument({ kind: "html", title: "Totals ", html, layout: "wide" }, { scheme: "dark", preset: "dusk" }, BUNDLE, LIBRARIES); + + // Exactly the payload and the viewer script close a script element: the + // model's markup stays inside the JSON string. + assert.equal(document.match(/<\/script>/gu)?.length, 2); + assert.match(document, new RegExp(``, "u")); + assert.match(document, /Totals <b><\/title>/u); + + const payload = payloadOf(document); + assert.equal(payload.kind, "html"); + assert.deepEqual( + { title: payload.title, scheme: payload.scheme, preset: payload.preset, layout: payload.layout }, + { title: "Totals <b>", scheme: "dark", preset: "dusk", layout: "wide" }, + ); + const guest = payload.kind === "html" ? payload.guest : ""; + assert.ok(guest.includes(html.slice(0, 20)), "the guest keeps the model's markup"); + // The guest runs offline: its libraries are inline, nothing points at the desktop's custom protocol. + assert.doesNotMatch(guest, /aiden-genui:\/\//u); + assert.ok(guest.includes("CHART-SOURCE")); + assert.ok(!guest.includes("PLOTLY-SOURCE"), "Plotly is left out when the visual never calls it"); + assert.ok(!guest.includes("KATEX-SOURCE")); + assert.ok(guest.includes(".aiden-card"), "the component kit always ships"); +}); + +test("an HTML visual that draws with Plotly or KaTeX gets those libraries", () => { + const document = buildVisualViewerDocument( + { kind: "html", title: "Fit", html: `<div id="p"></div><script>Plotly.newPlot("p", []); katex.render("x", p);</script>` }, + LIGHT, + BUNDLE, + LIBRARIES, + ); + const payload = payloadOf(document); + const guest = payload.kind === "html" ? payload.guest : ""; + assert.ok(guest.includes("PLOTLY-SOURCE") && guest.includes("KATEX-SOURCE") && guest.includes("KATEX-STYLE")); + assert.ok(!guest.includes("CHART-SOURCE")); +}); + +test("a native visual carries its tree and only the images handed to it", () => { + const document = buildVisualViewerDocument( + { + kind: "ui", + title: "Plan options", + visual: UI_VISUAL, + attachments: [{ id: "photo-1", mimeType: "image/png", data: "AAAA", name: "ignored" } as never], + }, + LIGHT, + BUNDLE, + LIBRARIES, + ); + const payload = payloadOf(document); + assert.equal(payload.kind, "ui"); + assert.deepEqual(payload.kind === "ui" && payload.visual, UI_VISUAL); + assert.deepEqual(payload.kind === "ui" && payload.attachments, [{ id: "photo-1", mimeType: "image/png", data: "AAAA" }]); + assert.ok(!document.includes("CHART-SOURCE"), "native visuals draw with the viewer bundle, not the guest libraries"); + assert.ok(document.includes(BUNDLE.script) && document.includes(BUNDLE.style)); +}); + +test("documents refuse a bundle that could end its own element and anything over the size bound", () => { + const source = { kind: "ui" as const, title: "Plan", visual: UI_VISUAL, attachments: [] }; + assert.throws(() => buildVisualViewerDocument(source, LIGHT, { ...BUNDLE, script: "x='</script>'" }, LIBRARIES), /cannot be inlined/u); + assert.throws(() => buildVisualViewerDocument(source, LIGHT, { ...BUNDLE, style: "</style>" }, LIBRARIES), /cannot be inlined/u); + const huge = { ...source, attachments: [{ id: "big", mimeType: "image/png", data: "A".repeat(MAX_VISUAL_VIEWER_DOCUMENT_BYTES) }] }; + assert.throws(() => buildVisualViewerDocument(huge, LIGHT, BUNDLE, LIBRARIES), /too large/u); +}); + +test("the document ETag changes with the document and only with it", () => { + const first = buildVisualViewerDocument({ kind: "ui", title: "Plan", visual: UI_VISUAL, attachments: [] }, LIGHT, BUNDLE, LIBRARIES); + const again = buildVisualViewerDocument({ kind: "ui", title: "Plan", visual: UI_VISUAL, attachments: [] }, LIGHT, BUNDLE, LIBRARIES); + const dark = buildVisualViewerDocument({ kind: "ui", title: "Plan", visual: UI_VISUAL, attachments: [] }, { ...LIGHT, scheme: "dark" }, BUNDLE, LIBRARIES); + assert.equal(visualViewerDocumentEtag(first), visualViewerDocumentEtag(again)); + assert.notEqual(visualViewerDocumentEtag(first), visualViewerDocumentEtag(dark)); + assert.match(visualViewerDocumentEtag(first), /^"[A-Za-z0-9_-]{43}"$/u); +}); diff --git a/main/services/visual-viewer-document.ts b/main/services/visual-viewer-document.ts new file mode 100644 index 000000000..1b7442f61 --- /dev/null +++ b/main/services/visual-viewer-document.ts @@ -0,0 +1,158 @@ +import { createHash } from "node:crypto"; +import type { ChatUiVisualV1 } from "../../renderer/shared/aiden-ui/types.js"; +import { THEME_PRESETS, type AppearanceScheme, type ThemePresetId } from "../../renderer/shared/appearance.js"; +import type { GENERATIVE_UI_HOST_LIBS } from "../../renderer/shared/generative-ui.js"; +import { + VISUAL_VIEWER_CSP, + VISUAL_VIEWER_PAYLOAD_ELEMENT_ID, + type VisualViewerPayload, +} from "../../renderer/shared/visual-viewer.js"; +import { inlineGenerativeUiHostLibraries, wrapGenerativeUiHtml } from "./generative-ui-html.js"; + +/** + * Builds the self-contained document phones open for one visual (Remote + * contract revision 34, `chat-visual-viewer-v1`). Electron-free: the viewer + * bundle and host libraries are passed in. + */ + +/** The built viewer page (`build/renderer/visual-viewer/`), read once by main. */ +export interface VisualViewerBundle { + script: string; + style: string; +} + +export const MAX_VISUAL_VIEWER_DOCUMENT_BYTES = 12 * 1024 * 1024; + +export interface VisualViewerAppearance { + scheme: AppearanceScheme; + preset: ThemePresetId; +} + +export type VisualViewerSource = + | { kind: "html"; title: string; html: string; layout?: "wide" } + | { + kind: "ui"; + title: string; + visual: ChatUiVisualV1; + attachments: readonly { id: string; mimeType: string; data: string }[]; + layout?: "wide"; + }; + +const PRESET_IDS = new Set<string>(THEME_PRESETS.map((preset) => preset.id)); + +/** + * Reads the document query. Both keys are optional; a preset this host does + * not know falls back to Aiden's own, so a phone with a newer catalog still + * gets a document. Anything else is a bad request. + */ +export function parseVisualViewerQuery(query: URLSearchParams): VisualViewerAppearance { + for (const key of query.keys()) { + if (key !== "scheme" && key !== "theme") throw new Error(`Unknown visual document parameter ${key}.`); + } + if (query.getAll("scheme").length > 1 || query.getAll("theme").length > 1) { + throw new Error("Visual document parameters may appear once."); + } + const scheme = query.get("scheme") ?? "light"; + if (scheme !== "light" && scheme !== "dark") throw new Error("Visual document scheme must be light or dark."); + const theme = query.get("theme") ?? "aiden"; + if (!/^[a-z][a-z0-9-]{0,31}$/u.test(theme)) throw new Error("Visual document theme is invalid."); + return { scheme, preset: PRESET_IDS.has(theme) ? (theme as ThemePresetId) : "aiden" }; +} + +/** + * Host libraries an HTML visual can reach. Each one is large (Plotly alone is + * several MB), so a phone document carries only those its source names. The + * component kit stylesheet is small and always included. + */ +function referencedLibrary(html: string): (name: (typeof GENERATIVE_UI_HOST_LIBS)[number]) => boolean { + const usesChart = /\bChart\b/u.test(html); + const usesPlotly = /\bPlotly\b/u.test(html); + const usesKatex = /katex/iu.test(html); + return (name) => { + switch (name) { + case "chart.js": + return usesChart; + case "plotly.js": + return usesPlotly; + case "katex.js": + case "katex.css": + return usesKatex; + case "aiden-ui.css": + return true; + } + }; +} + +/** JSON that is safe inside `<script type="application/json">`. */ +function scriptSafeJson(value: unknown): string { + return JSON.stringify(value) + .replace(/</gu, "\\u003c") + .replace(/>/gu, "\\u003e") + .replace(/&/gu, "\\u0026") + .replace(/\u2028/gu, "\\u2028") + .replace(/\u2029/gu, "\\u2029"); +} + +function escapeText(value: string): string { + return value.replace(/&/gu, "&").replace(/</gu, "<").replace(/>/gu, ">"); +} + +export function buildVisualViewerDocument( + source: VisualViewerSource, + appearance: VisualViewerAppearance, + bundle: VisualViewerBundle, + libraries: Readonly<Record<string, string>>, +): string { + // The bundle is inlined verbatim, so it must not be able to end its own element. + if (/<\/script/iu.test(bundle.script) || /<\/style/iu.test(bundle.style)) { + throw new Error("The visual viewer bundle cannot be inlined."); + } + const base = { + version: 1 as const, + title: source.title, + scheme: appearance.scheme, + preset: appearance.preset, + ...(source.layout === "wide" ? { layout: "wide" as const } : {}), + }; + const payload: VisualViewerPayload = source.kind === "ui" + ? { ...base, kind: "ui", visual: source.visual, attachments: source.attachments.map(({ id, mimeType, data }) => ({ id, mimeType, data })) } + : { + ...base, + kind: "html", + guest: inlineGenerativeUiHostLibraries( + wrapGenerativeUiHtml(source.html, source.title, undefined, { inline: true }), + libraries, + referencedLibrary(source.html), + ), + }; + const document = `<!DOCTYPE html> +<html lang="en" data-color-scheme="${appearance.scheme}"> +<head> +<meta charset="utf-8"> +<meta http-equiv="Content-Security-Policy" content="${VISUAL_VIEWER_CSP}"> +<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover"> +<meta name="color-scheme" content="${appearance.scheme}"> +<title>${escapeText(source.title)} + + + +
+ + + + +`; + if (Buffer.byteLength(document, "utf8") > MAX_VISUAL_VIEWER_DOCUMENT_BYTES) { + throw new Error("This visual is too large to open on a phone."); + } + return document; +} + +/** A strong validator for the exact document bytes. */ +export function visualViewerDocumentEtag(document: string): string { + return `"${createHash("sha256").update(document).digest("base64url")}"`; +} diff --git a/main/services/visual-viewer-main.ts b/main/services/visual-viewer-main.ts new file mode 100644 index 000000000..33cb511e4 --- /dev/null +++ b/main/services/visual-viewer-main.ts @@ -0,0 +1,46 @@ +import { existsSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { getRendererBuildPath } from "../windows/window-paths.js"; +import { loadGenerativeUiHostLibraries } from "./generative-ui-host-libraries.js"; +import type { VisualViewerBundle } from "./visual-viewer-document.js"; + +/** + * Loads the phone visual viewer's built page and the host libraries its HTML + * guests inline. Both are immutable for a running app, so each is read once. + * A failed read is not cached, so a later request can retry. + */ + +const SCRIPT = ["visual-viewer", "viewer.js"] as const; +const STYLE = ["visual-viewer", "viewer.css"] as const; + +/** True when this build ships the viewer (`npm run build:visual-viewer`). */ +export function visualViewerInstalled(): boolean { + return existsSync(getRendererBuildPath(...SCRIPT)) && existsSync(getRendererBuildPath(...STYLE)); +} + +let bundle: Promise | undefined; +let libraries: Promise> | undefined; + +function once(load: () => Promise, read: () => Promise | undefined, store: (value: Promise | undefined) => void): Promise { + const cached = read(); + if (cached) return cached; + const pending = load(); + store(pending); + pending.catch(() => store(undefined)); + return pending; +} + +export function loadVisualViewerBundle(): Promise { + return once( + async () => ({ + script: await readFile(getRendererBuildPath(...SCRIPT), "utf8"), + style: await readFile(getRendererBuildPath(...STYLE), "utf8"), + }), + () => bundle, + (value) => { bundle = value; }, + ); +} + +export function loadVisualViewerLibraries(): Promise> { + return once(loadGenerativeUiHostLibraries, () => libraries, (value) => { libraries = value; }); +} diff --git a/main/windows/window-paths.ts b/main/windows/window-paths.ts index 966b954c0..974cb467b 100644 --- a/main/windows/window-paths.ts +++ b/main/windows/window-paths.ts @@ -17,3 +17,8 @@ export function getWindowUrl(htmlFileName: string): string { if (devServer) return `${devServer.replace(/\/$/, "")}/${htmlFileName}`; return pathToFileURL(path.join(buildRoot, "renderer", htmlFileName)).toString(); } + +/** A file the renderer build ships beside its pages (for example the phone visual viewer). */ +export function getRendererBuildPath(...segments: string[]): string { + return path.join(buildRoot, "renderer", ...segments); +} diff --git a/package.json b/package.json index 3f1db83b1..8aba98e7a 100644 --- a/package.json +++ b/package.json @@ -30,10 +30,11 @@ "type": "module", "main": "build/main/index.js", "scripts": { - "build": "npm run build:native:c && vite build && node scripts/verify-gemini-live-worklet.mjs && node scripts/verify-ghostty-terminal-assets.mjs && npm run build:electron:production", + "build": "npm run build:native:c && vite build && npm run build:visual-viewer && node scripts/verify-gemini-live-worklet.mjs && node scripts/verify-ghostty-terminal-assets.mjs && npm run build:electron:production", "computer-use:vendor": "node scripts/vendor-cua-driver.mjs", "build:native": "npm run build:native:c && node scripts/build-foundation-models-helper.mjs --required && node scripts/build-cua-s1-forms-helper.mjs --required", "build:native:optional": "npm run build:native:c && node scripts/build-foundation-models-helper.mjs --optional && node scripts/build-cua-s1-forms-helper.mjs --optional", + "build:visual-viewer": "vite build --config vite.visual-viewer.config.ts", "build:bot-inbox-writer": "node scripts/build-bot-inbox-writer.mjs", "build:subagent-file-mutator": "node scripts/build-subagent-file-mutator.mjs", "build:subagent-shell-runner": "node scripts/build-subagent-shell-runner.mjs", @@ -51,7 +52,7 @@ "cli:build": "npm --prefix packages/cli run build", "test:cli": "npm --prefix packages/cli ci && npm run cli:build && npm --prefix packages/cli test", "generative-ui:vendor": "node scripts/vendor-generative-ui-libs.mjs", - "pretest:generative-ui": "npm run build:subagent-file-mutator", + "pretest:generative-ui": "npm run build:subagent-file-mutator && npm run build:visual-viewer", "test:aiden-remote-speech": "tsx --test main/services/aiden-remote-speech.test.ts", "test:stt-smoke": "AIDEN_STT_SMOKE=1 tsx --test scripts/stt-smoke.test.ts", "test": "node scripts/run-ci-tests.mjs --parallel", @@ -63,7 +64,7 @@ "test:remote-chat": "tsx --test renderer/lib/hosts/remote-chat-mapper.test.ts renderer/lib/hosts/remote-stream-translator.test.ts main/services/peer-remote-chat-view.test.ts renderer/main/remote-chat-view.test.tsx renderer/lib/hosts/chat-session-control.test.ts renderer/lib/hosts/local-host-adapter.test.ts renderer/lib/hosts/observed-local-run.test.ts main/services/peer-remote-chat-control.test.ts renderer/lib/hosts/remote-new-chat.test.ts renderer/lib/hosts/host-resources.test.ts renderer/lib/hosts/new-chat-targets.test.ts renderer/main/remote-new-chat-view.test.tsx main/services/peer-remote-new-chat.test.ts main/services/peer-remote-chat-fork.test.ts", "test:peer-hosts": "tsx --test main/services/peer-intent-store.test.ts main/handlers/peer-intents.test.ts main/services/peer-host-registry.test.ts main/services/peer-transport.test.ts main/services/peer-operation.test.ts main/services/peer-host-feed-cache.test.ts main/services/peer-run-subscriptions.test.ts main/services/peer-host-manager.test.ts main/handlers/peer-host-live.test.ts main/services/peer-bootstrap-transport.test.ts main/services/peer-host-pairing.test.ts main/services/peer-pairing-client.test.ts main/services/peer-discovery.test.ts main/handlers/peer-pairing.test.ts main/services/peer-lan-addresses.test.ts main/services/peer-multi-host-acceptance.test.ts", "pretest:aiden-remote": "npm run build:worktree-file-io", - "test:aiden-remote": "npm run test:peer-hosts && tsx --test main/services/aiden-remote-chat-progress.test.ts main/services/aiden-remote-chat-progress-authorize.test.ts main/handlers/aiden-remote.test.ts main/services/aiden-remote-chat-summaries.test.ts main/services/aiden-remote-approved-roots.test.ts main/services/aiden-remote-revocation.test.ts main/services/aiden-remote-bot-files.test.ts main/services/aiden-remote-bots.test.ts main/services/aiden-remote-bot-session.test.ts main/services/aiden-remote-chat-http.test.ts main/services/aiden-remote-chats.test.ts main/services/aiden-remote-files.test.ts main/services/aiden-remote-git.test.ts main/services/aiden-remote-models.test.ts main/services/aiden-remote-protocol.test.ts main/services/aiden-remote-opaque-handles.test.ts main/services/aiden-remote-operation-contract.test.ts main/services/aiden-remote-pairing.test.ts main/services/aiden-remote-pairing-requests.test.ts main/services/aiden-remote-pairing-requests-router.test.ts main/services/aiden-remote-pairing-request-host.test.ts main/services/aiden-remote-sealed-envelope.test.ts main/services/aiden-remote-ports.test.ts main/services/aiden-remote-router.test.ts main/services/aiden-remote-schedules.test.ts main/services/aiden-remote-service.test.ts main/services/aiden-remote-state.test.ts main/services/aiden-remote-streams.test.ts main/services/aiden-remote-tailscale-route.test.ts main/services/aiden-remote-tailscale.test.ts main/services/aiden-remote-tls-identity.test.ts main/services/aiden-remote-workspace-browser.test.ts main/services/aiden-remote-workspace-http.test.ts main/services/aiden-remote-workspaces.test.ts renderer/components/remote-connection-popover.test.tsx renderer/components/remote-pairing-request-sheet.test.tsx renderer/components/settings/remote-access-settings.test.tsx renderer/lib/remote-approval.test.ts renderer/lib/remote-connection-status.test.ts renderer/lib/remote-pairing-lifecycle.test.ts renderer/lib/settings-section.test.ts main/services/run-event-projection.test.ts main/services/host-run-registry.test.ts main/services/host-runs.test.ts main/services/aiden-remote-host-feed.test.ts main/services/aiden-remote-host-runs.test.ts main/handlers/local-runs.test.ts main/services/repository-identity.test.ts && node --test scripts/aiden-remote-lan-transport-spike.test.mjs && tsx --test renderer/lib/peer-connections.test.ts renderer/components/settings/connections-settings.test.tsx", + "test:aiden-remote": "npm run test:peer-hosts && tsx --test main/services/aiden-remote-chat-progress.test.ts main/services/aiden-remote-chat-progress-authorize.test.ts main/handlers/aiden-remote.test.ts main/services/aiden-remote-chat-summaries.test.ts main/services/aiden-remote-approved-roots.test.ts main/services/aiden-remote-revocation.test.ts main/services/aiden-remote-bot-files.test.ts main/services/aiden-remote-bots.test.ts main/services/aiden-remote-bot-session.test.ts main/services/aiden-remote-chat-http.test.ts main/services/aiden-remote-chats.test.ts main/services/visual-viewer-document.test.ts main/services/aiden-remote-files.test.ts main/services/aiden-remote-git.test.ts main/services/aiden-remote-models.test.ts main/services/aiden-remote-protocol.test.ts main/services/aiden-remote-opaque-handles.test.ts main/services/aiden-remote-operation-contract.test.ts main/services/aiden-remote-pairing.test.ts main/services/aiden-remote-pairing-requests.test.ts main/services/aiden-remote-pairing-requests-router.test.ts main/services/aiden-remote-pairing-request-host.test.ts main/services/aiden-remote-sealed-envelope.test.ts main/services/aiden-remote-ports.test.ts main/services/aiden-remote-router.test.ts main/services/aiden-remote-schedules.test.ts main/services/aiden-remote-service.test.ts main/services/aiden-remote-state.test.ts main/services/aiden-remote-streams.test.ts main/services/aiden-remote-tailscale-route.test.ts main/services/aiden-remote-tailscale.test.ts main/services/aiden-remote-tls-identity.test.ts main/services/aiden-remote-workspace-browser.test.ts main/services/aiden-remote-workspace-http.test.ts main/services/aiden-remote-workspaces.test.ts renderer/components/remote-connection-popover.test.tsx renderer/components/remote-pairing-request-sheet.test.tsx renderer/components/settings/remote-access-settings.test.tsx renderer/lib/remote-approval.test.ts renderer/lib/remote-connection-status.test.ts renderer/lib/remote-pairing-lifecycle.test.ts renderer/lib/settings-section.test.ts main/services/run-event-projection.test.ts main/services/host-run-registry.test.ts main/services/host-runs.test.ts main/services/aiden-remote-host-feed.test.ts main/services/aiden-remote-host-runs.test.ts main/handlers/local-runs.test.ts main/services/repository-identity.test.ts && node --test scripts/aiden-remote-lan-transport-spike.test.mjs && tsx --test renderer/lib/peer-connections.test.ts renderer/components/settings/connections-settings.test.tsx", "test:aiden-remote-chat-summaries": "tsx --test main/services/aiden-remote-chat-summaries.test.ts", "test:memory-policy": "tsx --test main/services/memory-policy.test.ts main/services/aiden-remote-memory-settings.test.ts renderer/components/settings/memory-settings.test.tsx", "test:aiden-service-boundary": "tsx --test main/services/empty-chat-migration.test.ts main/services/chat-first-message-commit.test.ts main/services/chat-application-service.test.ts main/services/chat-generation-owner.test.ts main/services/workspace-application-service.test.ts main/services/workspace-environment-application-service.test.ts main/services/workspace-worktree-application-service.test.ts main/services/scheduled-task-application-service.test.ts main/services/bot-application-service.test.ts", diff --git a/protocol/aiden-remote/v1/fixtures/contract.json b/protocol/aiden-remote/v1/fixtures/contract.json index 35096aed0..a551154c0 100644 --- a/protocol/aiden-remote/v1/fixtures/contract.json +++ b/protocol/aiden-remote/v1/fixtures/contract.json @@ -1,5 +1,5 @@ { - "contractRevision": 33, + "contractRevision": 34, "peerRouteAdvertisement": {"feature":"peer-routes-v1","routes":[{"endpoint":"https://fixture-peer.example.ts.net/api/aiden/v1","serverSpkiSha256":"sha256/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="}]}, "providerCreation": {"label":"Fixture private model","baseUrl":"https://models.example.test/v1","kind":"openai","deployment":"hosted","needsKey":false,"models":[{"id":"fixture-vision","vision":true,"reasoning":false,"toolCall":true}],"confirmedForeground":true}, "providerCreationReceipt": {"id":"custom:remote-fixture","label":"Fixture private model","models":["fixture-vision"]}, @@ -1537,7 +1537,7 @@ "instanceId": "instance_fixture_01", "displayName": "Fixture Aiden", "platform": "mac", - "contractRevision": 33, + "contractRevision": 34, "pairingRequests": true }, "hostFeedEvents": [ diff --git a/protocol/aiden-remote/v1/openapi.json b/protocol/aiden-remote/v1/openapi.json index e46650a20..838269098 100644 --- a/protocol/aiden-remote/v1/openapi.json +++ b/protocol/aiden-remote/v1/openapi.json @@ -1510,6 +1510,77 @@ } } }, + "/chats/{chatId}/visuals/{visualId}/document": { + "parameters": [ + { + "$ref": "#/components/parameters/ProtocolVersion" + } + ], + "get": { + "operationId": "getChatVisualDocument", + "x-aiden-archived-access": "readable", + "x-aiden-capability": "chat:read", + "x-aiden-feature": "chat-visual-viewer-v1", + "description": "Contract revision 34. One self-contained HTML document that draws a visual a message in this chat projects, for phones to open full screen in a locked-down web view. Announced by `chat-visual-viewer-v1`; visual entries gain no new key. The document has no subresources and talks to its host only through the native message channel (iOS `aidenVisual` handler, Android `AidenVisualBridge`) with `ready`, `prompt` and `openUrl` messages. Served `no-store` with a strong ETag (a matching `If-None-Match` gets 304) and gzip when accepted.", + "parameters": [ + { + "$ref": "#/components/parameters/ChatId" + }, + { + "name": "visualId", + "in": "path", + "required": true, + "schema": { + "type": "string", + "pattern": "^[A-Za-z0-9._:-]{1,256}$" + } + }, + { + "name": "scheme", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "light", + "dark" + ], + "default": "light" + } + }, + { + "name": "theme", + "in": "query", + "required": false, + "description": "A preset id from aiden-appearance-v1.json. An id this host does not know falls back to aiden.", + "schema": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]{0,31}$", + "default": "aiden" + } + } + ], + "responses": { + "200": { + "description": "The viewer document", + "content": { + "text/html": { + "schema": { + "type": "string", + "maxLength": 12582912 + } + } + } + }, + "304": { + "description": "The document matches If-None-Match" + }, + "default": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/bots": { "parameters": [ { diff --git a/renderer/components/aiden-ui/aiden-ui-block.tsx b/renderer/components/aiden-ui/aiden-ui-block.tsx index 6cd327f8c..966d424d4 100644 --- a/renderer/components/aiden-ui/aiden-ui-block.tsx +++ b/renderer/components/aiden-ui/aiden-ui-block.tsx @@ -92,6 +92,11 @@ export interface AidenUiBlockProps { hideCaption?: boolean; /** A reply is running: confirming a follow-up adds it to the draft instead. */ followUpBusy?: boolean; + /** + * Hosts that confirm follow-ups in their own UI (the phone viewer) get the + * requested text here instead of the inline Send chip. + */ + onFollowUp?: (text: string) => void; } /** @@ -122,6 +127,7 @@ function AidenUiVisualBody({ attachments, hideCaption = false, followUpBusy = false, + onFollowUp, }: AidenUiBlockProps) { const { openLink } = useLinkOpener(); const [state, setLocalState] = React.useState>(() => initialState(visual)); @@ -145,10 +151,12 @@ function AidenUiVisualBody({ const data = React.useMemo(() => parseData(visual.dataJson), [visual.dataJson]); const onActionRef = React.useRef(onAction); const onStateChangeRef = React.useRef(onStateChange); + const onFollowUpRef = React.useRef(onFollowUp); React.useLayoutEffect(() => { onActionRef.current = onAction; onStateChangeRef.current = onStateChange; - }, [onAction, onStateChange]); + onFollowUpRef.current = onFollowUp; + }, [onAction, onStateChange, onFollowUp]); const setState = React.useCallback( (key: string, value: unknown) => { @@ -182,7 +190,9 @@ function AidenUiVisualBody({ // Like HTML visuals, a follow-up never sends by itself: Aiden shows // the exact text and only the user's confirmation sends it. const text = asString(evaluate(action.text, scope)).trim(); - if (text && text.length <= MAX_GUEST_PROMPT_CHARS) setPendingFollowUp(text); + if (!text || text.length > MAX_GUEST_PROMPT_CHARS) return; + if (onFollowUpRef.current) onFollowUpRef.current(text); + else setPendingFollowUp(text); return; } case "copy": { diff --git a/renderer/lib/link-opener.tsx b/renderer/lib/link-opener.tsx index c41b98f97..d65d6fc34 100644 --- a/renderer/lib/link-opener.tsx +++ b/renderer/lib/link-opener.tsx @@ -48,6 +48,28 @@ export function useLinkOpener(): Omit boolean }>) { + const value = React.useMemo( + () => ({ + openLink: (href) => openExternal(href), + canOpen: (href) => /^https:\/\//iu.test(href), + preference: "external", + registerSurface: () => () => undefined, + registerBlocker: () => () => undefined, + }), + [openExternal], + ); + return {children}; +} + /** Mount while the browser panel can actually be shown on this route. */ export function useBrowserSurface(active: boolean): void { const { registerSurface } = React.useContext(LinkOpenerContext); diff --git a/renderer/shared/visual-viewer.ts b/renderer/shared/visual-viewer.ts new file mode 100644 index 000000000..da5bf025a --- /dev/null +++ b/renderer/shared/visual-viewer.ts @@ -0,0 +1,85 @@ +/** + * The phone visual viewer (Remote contract revision 34, `chat-visual-viewer-v1`). + * + * The Mac serves one self-contained HTML document per visual; iOS and Android + * load it into a locked-down web view. The document carries its payload as + * JSON and talks to the native host only through one message channel. This + * module is shared by the document builder in main and the viewer page. + */ +import type { ChatUiVisualV1 } from "./aiden-ui/types.js"; +import type { AppearanceScheme, ThemePresetId } from "./appearance.js"; + +export const VISUAL_VIEWER_PAYLOAD_ELEMENT_ID = "aiden-visual-payload" as const; +/** iOS: `window.webkit.messageHandlers.aidenVisual.postMessage(object)`. */ +export const VISUAL_VIEWER_IOS_HANDLER = "aidenVisual" as const; +/** Android: `window.AidenVisualBridge.postMessage(jsonString)`. */ +export const VISUAL_VIEWER_ANDROID_BRIDGE = "AidenVisualBridge" as const; + +/** + * The viewer page runs only its own inline bundle and frames only its HTML + * guest's srcdoc. Nothing loads from the network. The guest's own CSP (inside + * its srcdoc) still applies on top of this one. + */ +export const VISUAL_VIEWER_CSP = + "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; img-src data: blob:; font-src data:; connect-src 'none'; frame-src about:srcdoc; child-src about:srcdoc; object-src 'none'; base-uri 'none'; form-action 'none'; worker-src 'none'; media-src data:"; + +interface VisualViewerPayloadBase { + version: 1; + title: string; + scheme: AppearanceScheme; + preset: ThemePresetId; + layout?: "wide"; +} + +export type VisualViewerPayload = + | (VisualViewerPayloadBase & { + kind: "ui"; + visual: ChatUiVisualV1; + /** The message images the visual shows, as base64 data. */ + attachments: { id: string; mimeType: string; data: string }[]; + }) + | (VisualViewerPayloadBase & { + kind: "html"; + /** The wrapped guest document (host libraries inlined), framed as srcdoc. */ + guest: string; + }); + +/** Messages the page sends to its native host. A closed set: hosts ignore anything else. */ +export type VisualViewerNativeMessage = + | { type: "ready" } + | { type: "prompt"; text: string } + | { type: "openUrl"; url: string }; + +interface NativeChannels { + webkit?: { messageHandlers?: Record }; + AidenVisualBridge?: { postMessage(value: string): void }; +} + +/** + * Sends a message to whichever native host loaded the page. Returns false in a + * plain browser (no channel), so callers can fall back or do nothing. + */ +export function postVisualViewerMessage(message: VisualViewerNativeMessage, target: unknown = globalThis): boolean { + const channels = target as NativeChannels; + const ios = channels.webkit?.messageHandlers?.[VISUAL_VIEWER_IOS_HANDLER]; + if (ios) { + ios.postMessage(message); + return true; + } + const android = channels[VISUAL_VIEWER_ANDROID_BRIDGE]; + if (android) { + android.postMessage(JSON.stringify(message)); + return true; + } + return false; +} + +/** The only links a visual may ask its host to open. */ +export function isViewerOpenableUrl(value: string): boolean { + try { + const url = new URL(value); + return url.protocol === "https:" && url.username === "" && url.password === ""; + } catch { + return false; + } +} diff --git a/renderer/visual-viewer/main.tsx b/renderer/visual-viewer/main.tsx new file mode 100644 index 000000000..8336d5eb1 --- /dev/null +++ b/renderer/visual-viewer/main.tsx @@ -0,0 +1,198 @@ +import "../styles.css"; +import * as React from "react"; +import { createRoot } from "react-dom/client"; +import { AidenUiBlock } from "../components/aiden-ui/aiden-ui-block"; +import { Toaster } from "../components/ui"; +import { ExternalLinkOpenerProvider } from "../lib/link-opener"; +import { readGenerativeUiTheme } from "../lib/generative-ui-theme-snapshot"; +import { + createDefaultAppearanceConfig, + getPresetVariant, + resolveThemeTokens, + uiFontStack, + codeFontStack, + type AppearanceScheme, + type ThemePresetId, +} from "../shared/appearance"; +import { parseChatUiVisualV1 } from "../shared/aiden-ui/visual"; +import { + GENERATIVE_UI_THEME_MESSAGE, + admitGuestPrompt, + parseGuestBridgeMessage, +} from "../shared/generative-ui-bridge"; +import { GENERATIVE_UI_IFRAME_SANDBOX } from "../shared/generative-ui"; +import { + VISUAL_VIEWER_PAYLOAD_ELEMENT_ID, + isViewerOpenableUrl, + postVisualViewerMessage, + type VisualViewerPayload, +} from "../shared/visual-viewer"; + +/** + * The phone visual viewer. The Mac builds this page with its payload inline + * and iOS or Android loads it into a locked-down web view. It has no preload, + * no storage and no network; it reaches its host only through + * `postVisualViewerMessage`. + */ + +const MIN_GUEST_HEIGHT = 64; +const MAX_GUEST_HEIGHT = 20_000; +const CHART_WAIT_MS = 2500; + +function readPayload(): VisualViewerPayload | undefined { + const element = document.getElementById(VISUAL_VIEWER_PAYLOAD_ELEMENT_ID); + if (!element?.textContent) return undefined; + try { + return JSON.parse(element.textContent) as VisualViewerPayload; + } catch { + return undefined; + } +} + +/** The preset's tokens for one scheme, applied without the app's storage-backed runtime. */ +function applyViewerTheme(scheme: AppearanceScheme, preset: ThemePresetId): void { + const defaults = createDefaultAppearanceConfig(); + const variant = getPresetVariant(preset, scheme); + const root = document.documentElement; + root.classList.toggle("dark", scheme === "dark"); + root.style.colorScheme = scheme; + root.dataset.appearanceScheme = scheme; + root.dataset.theme = variant.preset; + const reduceMotion = window.matchMedia?.("(prefers-reduced-motion: reduce)").matches === true; + root.dataset.reduceMotion = String(reduceMotion); + root.style.setProperty("--font-ui-family", uiFontStack(variant.uiFont)); + root.style.setProperty("--font-code-family", codeFontStack(variant.codeFont)); + root.style.setProperty("--ui-font-size", `${defaults.uiFontSize}px`); + root.style.setProperty("--code-font-size", `${defaults.codeFontSize}px`); + for (const [name, token] of Object.entries(resolveThemeTokens(variant, scheme))) { + root.style.setProperty(name, token); + } +} + +const openExternal = (url: string): boolean => + isViewerOpenableUrl(url) && postVisualViewerMessage({ type: "openUrl", url }); + +function UiViewer({ payload }: { payload: Extract }) { + const visual = React.useMemo(() => parseChatUiVisualV1(payload.visual), [payload.visual]); + React.useEffect(() => { + let live = true; + // Ready once every chart has drawn (or the wait runs out). + const deadline = Date.now() + CHART_WAIT_MS; + const check = () => { + if (!live) return; + const canvases = [...document.querySelectorAll("canvas[role='img']")]; + if (canvases.every((canvas) => canvas.dataset.chartReady === "true") || Date.now() > deadline) { + postVisualViewerMessage({ type: "ready" }); + return; + } + setTimeout(check, 50); + }; + requestAnimationFrame(() => requestAnimationFrame(check)); + return () => { + live = false; + }; + }, []); + if (!visual) return

{payload.title}

; + return ( + + postVisualViewerMessage({ type: "prompt", text })} + /> + + ); +} + +function HtmlViewer({ payload }: { payload: Extract }) { + const frame = React.useRef(null); + const [height, setHeight] = React.useState(MIN_GUEST_HEIGHT); + // Hidden until the guest has the phone's theme, so a dark viewer never flashes light. + const [themed, setThemed] = React.useState(false); + const lastPrompt = React.useRef(undefined); + const announced = React.useRef(false); + + // A layout effect, so the listener is in place before the srcdoc guest parses + // and posts its first messages. + React.useLayoutEffect(() => { + const onMessage = (event: MessageEvent) => { + const guest = frame.current?.contentWindow; + if (!guest || event.source !== guest) return; + const message = parseGuestBridgeMessage(event.data); + if (!message) return; + switch (message.type) { + case "ready": { + const theme = readGenerativeUiTheme(); + guest.postMessage({ type: GENERATIVE_UI_THEME_MESSAGE, colorScheme: theme.colorScheme, vars: theme.vars }, "*"); + requestAnimationFrame(() => setThemed(true)); + return; + } + case "resize": { + if (!Number.isFinite(message.height)) return; + setHeight(Math.min(MAX_GUEST_HEIGHT, Math.max(MIN_GUEST_HEIGHT, Math.ceil(message.height)))); + return; + } + case "prompt": { + const now = performance.now(); + const decision = admitGuestPrompt({ text: message.text, now, lastAcceptedAt: lastPrompt.current }); + if (decision.action !== "confirm") return; + lastPrompt.current = now; + postVisualViewerMessage({ type: "prompt", text: decision.text }); + return; + } + case "escape": + return; + } + }; + window.addEventListener("message", onMessage); + return () => window.removeEventListener("message", onMessage); + }, []); + + React.useEffect(() => { + if (!themed || announced.current) return; + announced.current = true; + postVisualViewerMessage({ type: "ready" }); + }, [themed]); + + return ( +