diff --git a/apps/mobile/src/app/task/[id].tsx b/apps/mobile/src/app/task/[id].tsx index cd0a4c4843..6126da8b6a 100644 --- a/apps/mobile/src/app/task/[id].tsx +++ b/apps/mobile/src/app/task/[id].tsx @@ -704,6 +704,7 @@ export default function TaskDetailScreen() { small visual buffer at the bottom. */} ReactNode; +} + +export function MessageFileChip({ fileName }: { fileName: string }) { + const themeColors = useThemeColors(); + return ( + + + + {fileName} + + + ); } const COLLAPSED_MAX_HEIGHT = 160; @@ -34,6 +52,7 @@ export function HumanMessage({ content, timestamp, attachments, + renderAttachment, }: HumanMessageProps) { const themeColors = useThemeColors(); const [isExpanded, setIsExpanded] = useState(false); @@ -109,28 +128,15 @@ export function HumanMessage({ )} {hasAttachments && ( - {attachments?.map((att) => - att.kind === "image" ? ( - - ) : ( - - - - {att.fileName} - - - ), - )} + {attachments?.map((att) => ( + + {renderAttachment ? ( + renderAttachment(att) + ) : ( + + )} + + ))} )} diff --git a/apps/mobile/src/features/chat/index.ts b/apps/mobile/src/features/chat/index.ts index c582575cf1..f193530707 100644 --- a/apps/mobile/src/features/chat/index.ts +++ b/apps/mobile/src/features/chat/index.ts @@ -5,7 +5,12 @@ // Components export { AgentMessage } from "./components/AgentMessage"; -export { HumanMessage } from "./components/HumanMessage"; +export { + HumanMessage, + type HumanMessageAttachment, + MessageFileChip, +} from "./components/HumanMessage"; +export { MarkdownImage } from "./components/MarkdownImage"; export { MarkdownText } from "./components/MarkdownText"; export type { ToolKind, diff --git a/apps/mobile/src/features/tasks/api.test.ts b/apps/mobile/src/features/tasks/api.test.ts index 2ac4f23d57..ab54e96253 100644 --- a/apps/mobile/src/features/tasks/api.test.ts +++ b/apps/mobile/src/features/tasks/api.test.ts @@ -24,7 +24,12 @@ vi.mock("@/lib/api", () => ({ }), })); -import { cancelRun, HttpError, runTaskInCloud } from "./api"; +import { + cancelRun, + HttpError, + presignTaskRunArtifact, + runTaskInCloud, +} from "./api"; function bodyOf(call: unknown): Record { const [, init] = call as [string, RequestInit]; @@ -155,3 +160,39 @@ describe("cancelRun", () => { ); }); }); + +describe("presignTaskRunArtifact", () => { + beforeEach(() => { + mockFetch.mockReset(); + }); + + it("posts the storage path and returns the presigned URL", async () => { + mockFetch.mockResolvedValue( + new Response( + JSON.stringify({ url: "https://s3.example.com/x.png?sig=abc" }), + { status: 200 }, + ), + ); + + await expect( + presignTaskRunArtifact("task-1", "run-1", "tasks/run-1/artifacts/x.png"), + ).resolves.toBe("https://s3.example.com/x.png?sig=abc"); + + const [url, init] = mockFetch.mock.calls[0] as [string, RequestInit]; + expect(url).toBe( + "https://app.posthog.test/api/projects/42/tasks/task-1/runs/run-1/artifacts/presign/", + ); + expect(init.method).toBe("POST"); + expect(bodyOf(mockFetch.mock.calls[0])).toEqual({ + storage_path: "tasks/run-1/artifacts/x.png", + }); + }); + + it("throws an HttpError on a non-OK response", async () => { + mockFetch.mockResolvedValue(new Response("nope", { status: 500 })); + + await expect( + presignTaskRunArtifact("task-1", "run-1", "tasks/run-1/artifacts/x.png"), + ).rejects.toBeInstanceOf(HttpError); + }); +}); diff --git a/apps/mobile/src/features/tasks/api.ts b/apps/mobile/src/features/tasks/api.ts index fa703b7401..f7d03d1056 100644 --- a/apps/mobile/src/features/tasks/api.ts +++ b/apps/mobile/src/features/tasks/api.ts @@ -564,6 +564,38 @@ export async function getTaskRun( return await response.json(); } +/** + * Exchanges an artifact's storage path for a short-lived presigned S3 URL used + * to render image attachment previews. + */ +export async function presignTaskRunArtifact( + taskId: string, + runId: string, + storagePath: string, +): Promise { + const baseUrl = getBaseUrl(); + const projectId = getProjectId(); + + const response = await authedFetch( + `${baseUrl}/api/projects/${projectId}/tasks/${taskId}/runs/${runId}/artifacts/presign/`, + { + method: "POST", + body: JSON.stringify({ storage_path: storagePath }), + }, + ); + + if (!response.ok) { + throw new HttpError( + response.status, + response.statusText, + "Failed to generate artifact preview URL", + ); + } + + const data = (await response.json()) as { url: string }; + return data.url; +} + export async function cancelRun( taskId: string, runId: string, diff --git a/apps/mobile/src/features/tasks/components/CloudMessageAttachment.tsx b/apps/mobile/src/features/tasks/components/CloudMessageAttachment.tsx new file mode 100644 index 0000000000..da2d629a88 --- /dev/null +++ b/apps/mobile/src/features/tasks/components/CloudMessageAttachment.tsx @@ -0,0 +1,32 @@ +import { + type HumanMessageAttachment, + MarkdownImage, + MessageFileChip, +} from "@/features/chat"; +import { useCloudAttachmentPreview } from "../hooks/useCloudAttachmentPreview"; + +export function CloudMessageAttachment({ + attachment, + taskId, +}: { + attachment: HumanMessageAttachment; + taskId?: string; +}) { + const { data: previewUrl } = useCloudAttachmentPreview( + taskId, + attachment.cloudArtifact, + ); + + if (attachment.kind !== "image") { + return ; + } + + // Cloud images resolve to a presigned URL; local (in-flight) images render + // straight from their device uri. Fall back to a chip when neither is ready. + const imageUrl = attachment.cloudArtifact ? previewUrl : attachment.uri; + if (!imageUrl) { + return ; + } + + return ; +} diff --git a/apps/mobile/src/features/tasks/components/TaskSessionView.test.tsx b/apps/mobile/src/features/tasks/components/TaskSessionView.test.tsx index b89b8ceb7a..5bab080e15 100644 --- a/apps/mobile/src/features/tasks/components/TaskSessionView.test.tsx +++ b/apps/mobile/src/features/tasks/components/TaskSessionView.test.tsx @@ -51,6 +51,11 @@ vi.mock("./PlanApprovalCard", () => ({ createElement("PlanApprovalCard", props), })); +vi.mock("./CloudMessageAttachment", () => ({ + CloudMessageAttachment: (props: Record) => + createElement("CloudMessageAttachment", props), +})); + function renderTaskSessionView( props: Parameters[0], ): ReturnType { diff --git a/apps/mobile/src/features/tasks/components/TaskSessionView.tsx b/apps/mobile/src/features/tasks/components/TaskSessionView.tsx index fb4c23ed13..f30b4aec66 100644 --- a/apps/mobile/src/features/tasks/components/TaskSessionView.tsx +++ b/apps/mobile/src/features/tasks/components/TaskSessionView.tsx @@ -28,10 +28,13 @@ import type { SessionEvent, SessionNotification, SessionNotificationAttachment, + TerminalStatus, } from "../types"; +import { CloudMessageAttachment } from "./CloudMessageAttachment"; import { PlanApprovalCard } from "./PlanApprovalCard"; import { PlanStatusBar } from "./PlanStatusBar"; import { QuestionCard } from "./QuestionCard"; +import { TerminalStatusBanner } from "./TerminalStatusBanner"; interface PermissionResponseArgs { toolCallId: string; @@ -52,10 +55,11 @@ interface OptimisticUserMessage { interface TaskSessionViewProps { events: SessionEvent[]; + taskId?: string; pendingPermissions?: Record; isConnecting?: boolean; isThinking?: boolean; - terminalStatus?: "failed" | "completed"; + terminalStatus?: TerminalStatus; lastError?: string | null; onRetry?: () => void; onOpenTask?: (taskId: string) => void; @@ -797,6 +801,7 @@ function ConnectingIndicator() { export function TaskSessionView({ events, + taskId, pendingPermissions, isConnecting, isThinking, @@ -929,6 +934,13 @@ export function TaskSessionView({ [], ); + const renderAttachment = useCallback( + (attachment: SessionNotificationAttachment) => ( + + ), + [taskId], + ); + const renderMessage = useCallback( ({ item }: { item: ParsedMessage }) => { switch (item.type) { @@ -938,6 +950,7 @@ export function TaskSessionView({ content={item.content} timestamp={item.ts} attachments={item.attachments} + renderAttachment={renderAttachment} /> ); case "agent": @@ -994,7 +1007,12 @@ export function TaskSessionView({ return null; } }, - [onOpenTask, onSendPermissionResponse, pendingPermissions], + [ + onOpenTask, + onSendPermissionResponse, + pendingPermissions, + renderAttachment, + ], ); return ( @@ -1017,46 +1035,11 @@ export function TaskSessionView({ initialNumToRender={30} ListHeaderComponent={ terminalStatus ? ( - - - {terminalStatus === "failed" ? "Run failed" : "Run completed"} - - {lastError && ( - {lastError} - )} - {onRetry && ( - - - {terminalStatus === "failed" ? "Retry" : "Continue"} - - - )} - + ) : null } /> diff --git a/apps/mobile/src/features/tasks/components/TerminalStatusBanner.test.tsx b/apps/mobile/src/features/tasks/components/TerminalStatusBanner.test.tsx new file mode 100644 index 0000000000..13af28ea89 --- /dev/null +++ b/apps/mobile/src/features/tasks/components/TerminalStatusBanner.test.tsx @@ -0,0 +1,63 @@ +import { createElement } from "react"; +import { act, create } from "react-test-renderer"; +import { describe, expect, it, vi } from "vitest"; +import { + TerminalStatusBanner, + type TerminalStatusBannerProps, +} from "./TerminalStatusBanner"; + +function render(props: TerminalStatusBannerProps) { + let renderer!: ReturnType; + act(() => { + renderer = create(createElement(TerminalStatusBanner, props)); + }); + return renderer; +} + +function renderedText(renderer: ReturnType): string { + const acc: string[] = []; + const walk = (node: unknown) => { + if (typeof node === "string") { + acc.push(node); + } else if (Array.isArray(node)) { + node.forEach(walk); + } else if (node && typeof node === "object" && "children" in node) { + walk((node as { children: unknown }).children); + } + }; + walk(renderer.toJSON()); + return acc.join(" "); +} + +describe("TerminalStatusBanner", () => { + it.each([ + { terminalStatus: "completed", label: "Run completed", button: "Continue" }, + { terminalStatus: "failed", label: "Run failed", button: "Retry" }, + { terminalStatus: "stopped", label: "Run stopped", button: "Continue" }, + ] as const)( + "shows $label with a $button action for a $terminalStatus run", + ({ terminalStatus, label, button }) => { + const text = renderedText(render({ terminalStatus, onRetry: vi.fn() })); + expect(text).toContain(label); + expect(text).toContain(button); + }, + ); + + it("does not label a stopped run as failed", () => { + const text = renderedText(render({ terminalStatus: "stopped" })); + expect(text).not.toContain("Run failed"); + expect(text).not.toContain("Retry"); + }); + + it("fires onRetry when the action is pressed", () => { + const onRetry = vi.fn(); + const renderer = render({ terminalStatus: "stopped", onRetry }); + const pressable = renderer.root.findAll( + (node) => node.props.onPress === onRetry, + )[0]; + act(() => { + pressable.props.onPress(); + }); + expect(onRetry).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/mobile/src/features/tasks/components/TerminalStatusBanner.tsx b/apps/mobile/src/features/tasks/components/TerminalStatusBanner.tsx new file mode 100644 index 0000000000..aff1b0a166 --- /dev/null +++ b/apps/mobile/src/features/tasks/components/TerminalStatusBanner.tsx @@ -0,0 +1,57 @@ +import { Pressable, Text, View } from "react-native"; +import type { TerminalStatus } from "../types"; + +export interface TerminalStatusBannerProps { + terminalStatus: TerminalStatus; + lastError?: string | null; + onRetry?: () => void; +} + +export function TerminalStatusBanner({ + terminalStatus, + lastError, + onRetry, +}: TerminalStatusBannerProps) { + const isFailed = terminalStatus === "failed"; + const label = + terminalStatus === "failed" + ? "Run failed" + : terminalStatus === "stopped" + ? "Run stopped" + : "Run completed"; + + return ( + + + {label} + + {lastError && ( + {lastError} + )} + {onRetry && ( + + + {isFailed ? "Retry" : "Continue"} + + + )} + + ); +} diff --git a/apps/mobile/src/features/tasks/hooks/useCloudAttachmentPreview.ts b/apps/mobile/src/features/tasks/hooks/useCloudAttachmentPreview.ts new file mode 100644 index 0000000000..79af47a5c7 --- /dev/null +++ b/apps/mobile/src/features/tasks/hooks/useCloudAttachmentPreview.ts @@ -0,0 +1,47 @@ +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { getProjectId } from "@/lib/api"; +import { getTaskRun, presignTaskRunArtifact } from "../api"; +import type { CloudArtifactRef } from "../types"; + +// Presigned URLs outlive this comfortably (backend issues ~1h), so we refetch +// well before expiry rather than on every render. +const PREVIEW_STALE_MS = 50 * 60 * 1000; + +/** + * Resolves a cloud attachment to a presigned S3 preview URL. The run's artifact + * manifest is fetched once per run through the shared query cache, so a message + * with several images does not fire a manifest request per image. Returns + * `null` when the artifact is missing so callers can fall back to a file chip. + */ +export function useCloudAttachmentPreview( + taskId: string | undefined, + cloudArtifact: CloudArtifactRef | undefined, +) { + const queryClient = useQueryClient(); + const projectId = getProjectId(); + + return useQuery({ + queryKey: [ + "cloudArtifactPreview", + projectId, + taskId, + cloudArtifact?.runId, + cloudArtifact?.artifactId, + ], + enabled: Boolean(taskId && cloudArtifact), + staleTime: PREVIEW_STALE_MS, + retry: false, + queryFn: async () => { + if (!taskId || !cloudArtifact) return null; + const { runId, artifactId } = cloudArtifact; + const artifacts = await queryClient.fetchQuery({ + queryKey: ["taskRunArtifacts", projectId, taskId, runId], + queryFn: async () => (await getTaskRun(taskId, runId)).artifacts ?? [], + staleTime: PREVIEW_STALE_MS, + }); + const match = artifacts.find((artifact) => artifact.id === artifactId); + if (!match?.storage_path) return null; + return presignTaskRunArtifact(taskId, runId, match.storage_path); + }, + }); +} diff --git a/apps/mobile/src/features/tasks/stores/attachmentEchoStore.ts b/apps/mobile/src/features/tasks/stores/attachmentEchoStore.ts deleted file mode 100644 index 4a02ba4097..0000000000 --- a/apps/mobile/src/features/tasks/stores/attachmentEchoStore.ts +++ /dev/null @@ -1,65 +0,0 @@ -import AsyncStorage from "@react-native-async-storage/async-storage"; -import { create } from "zustand"; -import { createJSONStorage, persist } from "zustand/middleware"; -import type { SessionNotificationAttachment } from "../types"; - -/** - * Echoes of user messages that carried attachments, keyed by `taskRunId`. - * Persisted to disk so that re-entering a task — which discards the - * in-memory session and re-reads history from S3 — can still render the - * attachments the user sent locally. The cloud log doesn't surface attachment - * data on `user_message_chunk` events, so without this cache they would - * disappear after the screen unmounts. - * - * Entries are pushed in send-order. Re-hydration matches them positionally - * against the historical `user_message_chunk` events (Nth user message gets - * the Nth recorded echo) with a text-equality guard to degrade gracefully if - * the orders ever diverge. - */ -export interface AttachmentEcho { - text: string; - attachments: SessionNotificationAttachment[]; -} - -interface AttachmentEchoState { - echoes: Record; - recordEcho: ( - taskRunId: string, - text: string, - attachments: SessionNotificationAttachment[], - ) => void; - getEchoes: (taskRunId: string) => AttachmentEcho[]; - clearEchoes: (taskRunId: string) => void; -} - -export const useAttachmentEchoStore = create()( - persist( - (set, get) => ({ - echoes: {}, - recordEcho: (taskRunId, text, attachments) => { - if (attachments.length === 0) return; - set((state) => { - const existing = state.echoes[taskRunId] ?? []; - return { - echoes: { - ...state.echoes, - [taskRunId]: [...existing, { text, attachments }], - }, - }; - }); - }, - getEchoes: (taskRunId) => get().echoes[taskRunId] ?? [], - clearEchoes: (taskRunId) => { - set((state) => { - const { [taskRunId]: _, ...rest } = state.echoes; - return { echoes: rest }; - }); - }, - }), - { - name: "posthog-attachment-echoes", - storage: createJSONStorage(() => AsyncStorage), - partialize: (state) => ({ echoes: state.echoes }), - }, - ), -); diff --git a/apps/mobile/src/features/tasks/stores/taskSessionStore.test.ts b/apps/mobile/src/features/tasks/stores/taskSessionStore.test.ts index ddc5da1459..b0b49e1ef1 100644 --- a/apps/mobile/src/features/tasks/stores/taskSessionStore.test.ts +++ b/apps/mobile/src/features/tasks/stores/taskSessionStore.test.ts @@ -32,7 +32,11 @@ import type { TaskRun, } from "../types"; import { useMessageQueueStore } from "./messageQueueStore"; -import { type TaskSession, useTaskSessionStore } from "./taskSessionStore"; +import { + mapTerminalStatus, + type TaskSession, + useTaskSessionStore, +} from "./taskSessionStore"; import { useTaskStore } from "./taskStore"; function seedSession(overrides: Partial = {}): void { @@ -47,6 +51,20 @@ function seedSession(overrides: Partial = {}): void { useTaskSessionStore.setState({ sessions: { "run-1": session } }); } +describe("mapTerminalStatus", () => { + it.each([ + { status: "completed", expected: "completed" }, + { status: "failed", expected: "failed" }, + { status: "cancelled", expected: "stopped" }, + { status: "in_progress", expected: undefined }, + { status: "queued", expected: undefined }, + { status: undefined, expected: undefined }, + { status: null, expected: undefined }, + ] as const)("maps $status to $expected", ({ status, expected }) => { + expect(mapTerminalStatus(status)).toBe(expected); + }); +}); + describe("steerQueuedMessage", () => { beforeEach(() => { useMessageQueueStore.setState({ queuesByTaskId: {} }, false); diff --git a/apps/mobile/src/features/tasks/stores/taskSessionStore.ts b/apps/mobile/src/features/tasks/stores/taskSessionStore.ts index f7455157e5..e71f284ab5 100644 --- a/apps/mobile/src/features/tasks/stores/taskSessionStore.ts +++ b/apps/mobile/src/features/tasks/stores/taskSessionStore.ts @@ -27,11 +27,12 @@ import { type SessionNotificationAttachment, type StoredLogEntry, type Task, + type TerminalStatus, } from "../types"; import { convertStoredEntriesToEvents } from "../utils/parseSessionLogs"; import { playbackRateForTaskDuration } from "../utils/playbackRate"; +import { reinjectPromptAttachments } from "../utils/promptAttachments"; import { playCompletionSound } from "../utils/sounds"; -import { useAttachmentEchoStore } from "./attachmentEchoStore"; import { combineQueuedMessages, useMessageQueueStore, @@ -50,36 +51,6 @@ function completionPlaybackRate(promptStartedAt?: number): number { return playbackRateForTaskDuration(Date.now() - promptStartedAt); } -// Match historical `user_message_chunk` events (text-only, as the cloud -// stores them) against locally-cached attachment echoes by position+text. -// Echoes are written in send-order; we walk user messages in receive-order -// and zip them up. Drift (text mismatch at the same index) is treated as a -// no-op rather than a misattribution. -function reinjectAttachmentEchoes( - taskRunId: string, - events: SessionEvent[], -): void { - const echoes = useAttachmentEchoStore.getState().getEchoes(taskRunId); - if (echoes.length === 0) return; - - let echoIdx = 0; - for (const event of events) { - if (echoIdx >= echoes.length) return; - if (event.type !== "session_update") continue; - const update = event.notification?.update; - if (update?.sessionUpdate !== "user_message_chunk") continue; - if (update.attachments && update.attachments.length > 0) { - echoIdx++; - continue; - } - const echo = echoes[echoIdx]; - echoIdx++; - if (echo.text === (update.content?.text ?? "")) { - update.attachments = echo.attachments; - } - } -} - type LocalNotificationKind = | "turn_complete" | "awaiting_user_input" @@ -294,8 +265,8 @@ export interface TaskSession { // the log). Used to dedup the canonical copy against the echo. localUserEchoes?: Set; // Terminal backend status for this run, populated by status updates so the - // UI can surface "Run failed" / "Run completed". - terminalStatus?: "failed" | "completed"; + // UI can surface "Run failed" / "Run completed" / "Run stopped". + terminalStatus?: TerminalStatus; lastError?: string | null; // True when the user initiated work (new task, sendPrompt, resume) and // we should play a sound when control returns. False when reconnecting @@ -392,11 +363,12 @@ const connectAttempts = new Set(); // queue twice. const flushingTasks = new Set(); -function mapTerminalStatus( +export function mapTerminalStatus( status: string | undefined | null, -): "completed" | "failed" | undefined { +): TerminalStatus | undefined { if (status === "completed") return "completed"; - if (status === "failed" || status === "cancelled") return "failed"; + if (status === "failed") return "failed"; + if (status === "cancelled") return "stopped"; return undefined; } @@ -521,12 +493,6 @@ export const useTaskSessionStore = create((set, get) => ({ }, }, }; - if (echoAttachments.length > 0) { - useAttachmentEchoStore - .getState() - .recordEcho(session.taskRunId, prompt, echoAttachments); - } - set((state) => { const current = state.sessions[session.taskRunId]; const nextLocalEchoes = new Set(current.localUserEchoes ?? []); @@ -1024,10 +990,10 @@ export const useTaskSessionStore = create((set, get) => ({ : dedupAgainstLocalEchoes(update.newEntries, echoSet); const events = convertStoredEntriesToEvents(dedupedEntries); - // Snapshots are S3-backed and lose attachment metadata; reattach from - // the local echo store so historical user messages keep their images. + // Snapshots are S3-backed and replay user turns as text-only chunks; + // reattach the images from the `session/prompt` entries in the same log. if (isSnapshot) { - reinjectAttachmentEchoes(taskRunId, events); + reinjectPromptAttachments(events); } const analysis = analyzeEntries( diff --git a/apps/mobile/src/features/tasks/types.ts b/apps/mobile/src/features/tasks/types.ts index 18c31142ea..e777246331 100644 --- a/apps/mobile/src/features/tasks/types.ts +++ b/apps/mobile/src/features/tasks/types.ts @@ -45,6 +45,10 @@ export type TaskRunStatus = export const TERMINAL_STATUSES = ["completed", "failed", "cancelled"] as const; +// UI-facing terminal outcome for a run. `cancelled` maps to `stopped` (the user +// deliberately halted it) so the UI can distinguish it from a real failure. +export type TerminalStatus = "completed" | "failed" | "stopped"; + export function isTerminalStatus( status: TaskRunStatus | string | null | undefined, ): boolean { @@ -55,6 +59,11 @@ export function isTerminalStatus( ); } +export interface TaskRunArtifact { + id?: string; + storage_path?: string; +} + export interface TaskRun { id: string; task: string; @@ -68,6 +77,7 @@ export interface TaskRun { reasoning_effort?: string | null; output: Record | null; state: Record; + artifacts?: TaskRunArtifact[]; created_at: string; updated_at: string; completed_at: string | null; @@ -86,11 +96,20 @@ export interface StoredLogEntry { direction?: "client" | "agent"; } +export interface CloudArtifactRef { + runId: string; + artifactId: string; +} + export interface SessionNotificationAttachment { kind: "image" | "document"; uri: string; fileName: string; mimeType?: string; + // Set when the attachment was resolved from a cloud `session/prompt` entry. + // Its bytes live in S3 as a run artifact; the preview is fetched by presigning + // rather than read off the local device. + cloudArtifact?: CloudArtifactRef; } export interface SessionNotification { diff --git a/apps/mobile/src/features/tasks/utils/promptAttachments.test.ts b/apps/mobile/src/features/tasks/utils/promptAttachments.test.ts new file mode 100644 index 0000000000..c56aeb3528 --- /dev/null +++ b/apps/mobile/src/features/tasks/utils/promptAttachments.test.ts @@ -0,0 +1,210 @@ +import { describe, expect, it } from "vitest"; +import type { SessionEvent } from "../types"; +import { + extractSessionPromptAttachments, + parseCloudArtifactRef, + reinjectPromptAttachments, +} from "./promptAttachments"; + +const CLOUD_IMAGE_URI = + "file:///tmp/workspace/.posthog/attachments/run-123/artifact-456/screenshot.png"; +const CLOUD_DOC_URI = + "file:///tmp/workspace/.posthog/attachments/run-123/artifact-789/notes.pdf"; + +function promptEvent(prompt: unknown[]): SessionEvent { + return { + type: "acp_message", + direction: "client", + ts: 1, + message: { id: 1, method: "session/prompt", params: { prompt } }, + }; +} + +function userChunk(text: string): SessionEvent { + return { + type: "session_update", + ts: 2, + notification: { + update: { + sessionUpdate: "user_message_chunk", + content: { type: "text", text }, + }, + }, + }; +} + +describe("parseCloudArtifactRef", () => { + it.each([ + { + name: "resolves runId and artifactId", + pathname: + "/tmp/workspace/.posthog/attachments/run-123/artifact-456/x.png", + expected: { runId: "run-123", artifactId: "artifact-456" }, + }, + { + name: "ordinary file path is not a cloud artifact", + pathname: "/tmp/screenshot.png", + expected: undefined, + }, + { + name: "missing artifactId segment", + pathname: "/tmp/.posthog/attachments/run-123", + expected: undefined, + }, + { + name: "attachments dir without .posthog parent", + pathname: "/tmp/attachments/run-123/artifact-456/x.png", + expected: undefined, + }, + ])("$name", ({ pathname, expected }) => { + expect(parseCloudArtifactRef(pathname)).toEqual(expected); + }); +}); + +describe("extractSessionPromptAttachments", () => { + it("extracts a cloud image attachment from a resource_link block", () => { + const result = extractSessionPromptAttachments({ + method: "session/prompt", + params: { + prompt: [ + { type: "text", text: "what is this?" }, + { + type: "resource_link", + uri: CLOUD_IMAGE_URI, + name: "screenshot.png", + }, + ], + }, + }); + + expect(result).toEqual({ + text: "what is this?", + attachments: [ + { + kind: "image", + uri: CLOUD_IMAGE_URI, + fileName: "screenshot.png", + cloudArtifact: { runId: "run-123", artifactId: "artifact-456" }, + }, + ], + }); + }); + + it("marks non-image cloud attachments as documents", () => { + const result = extractSessionPromptAttachments({ + method: "session/prompt", + params: { prompt: [{ type: "resource_link", uri: CLOUD_DOC_URI }] }, + }); + + expect(result?.attachments[0]).toMatchObject({ + kind: "document", + fileName: "notes.pdf", + }); + }); + + it("ignores hidden text blocks when reconstructing prompt text", () => { + const result = extractSessionPromptAttachments({ + method: "session/prompt", + params: { + prompt: [ + { type: "text", text: "visible" }, + { type: "text", text: "hidden", _meta: { ui: { hidden: true } } }, + { + type: "resource_link", + uri: CLOUD_IMAGE_URI, + name: "screenshot.png", + }, + ], + }, + }); + + expect(result?.text).toBe("visible"); + }); + + it.each([ + { name: "non-prompt method", message: { method: "session/update" } }, + { + name: "prompt without attachments", + message: { + method: "session/prompt", + params: { prompt: [{ type: "text", text: "hi" }] }, + }, + }, + { + name: "ordinary file uri without cloud artifact path", + message: { + method: "session/prompt", + params: { prompt: [{ type: "image", uri: "file:///tmp/local.png" }] }, + }, + }, + ])("returns null for $name", ({ message }) => { + expect(extractSessionPromptAttachments(message)).toBeNull(); + }); +}); + +describe("reinjectPromptAttachments", () => { + it("reattaches attachments to the matching user_message_chunk", () => { + const events: SessionEvent[] = [ + promptEvent([ + { type: "text", text: "what is this?" }, + { type: "resource_link", uri: CLOUD_IMAGE_URI, name: "screenshot.png" }, + ]), + userChunk("what is this?"), + ]; + + reinjectPromptAttachments(events); + + const chunk = events[1]; + expect( + chunk.type === "session_update" && chunk.notification.update?.attachments, + ).toEqual([ + { + kind: "image", + uri: CLOUD_IMAGE_URI, + fileName: "screenshot.png", + cloudArtifact: { runId: "run-123", artifactId: "artifact-456" }, + }, + ]); + }); + + it("leaves ordinary user messages without attachments as-is", () => { + const events: SessionEvent[] = [userChunk("no attachments here")]; + + reinjectPromptAttachments(events); + + const chunk = events[0]; + expect( + chunk.type === "session_update" && chunk.notification.update?.attachments, + ).toBeUndefined(); + }); + + it("matches identical prompt texts in FIFO order", () => { + const secondUri = + "file:///tmp/workspace/.posthog/attachments/run-123/artifact-999/second.png"; + const events: SessionEvent[] = [ + promptEvent([ + { type: "text", text: "same" }, + { type: "resource_link", uri: CLOUD_IMAGE_URI, name: "first.png" }, + ]), + promptEvent([ + { type: "text", text: "same" }, + { type: "resource_link", uri: secondUri, name: "second.png" }, + ]), + userChunk("same"), + userChunk("same"), + ]; + + reinjectPromptAttachments(events); + + const first = events[2]; + const second = events[3]; + expect( + first.type === "session_update" && + first.notification.update?.attachments?.[0]?.fileName, + ).toBe("first.png"); + expect( + second.type === "session_update" && + second.notification.update?.attachments?.[0]?.fileName, + ).toBe("second.png"); + }); +}); diff --git a/apps/mobile/src/features/tasks/utils/promptAttachments.ts b/apps/mobile/src/features/tasks/utils/promptAttachments.ts new file mode 100644 index 0000000000..77f9d5be04 --- /dev/null +++ b/apps/mobile/src/features/tasks/utils/promptAttachments.ts @@ -0,0 +1,137 @@ +// Mirrors the artifact-ref parsing in @posthog/core/sessions/promptContent; +// mobile does not depend on @posthog/core, so the two are kept in sync by hand. +import { getFileName, isRasterImageFile } from "@posthog/shared"; +import type { + CloudArtifactRef, + SessionEvent, + SessionNotificationAttachment, +} from "../types"; + +interface PromptContentBlock { + type?: string; + text?: string; + uri?: string; + name?: string; + resource?: { uri?: string }; + _meta?: { ui?: { hidden?: boolean } }; +} + +interface PromptMessage { + method?: string; + params?: { prompt?: PromptContentBlock[] }; +} + +export interface PromptAttachmentGroup { + text: string; + attachments: SessionNotificationAttachment[]; +} + +// Cloud attachment bytes are uploaded as run artifacts and referenced from the +// stored prompt as `file://…/.posthog/attachments///`. +export function parseCloudArtifactRef( + pathname: string, +): CloudArtifactRef | undefined { + const segments = pathname.split("/").filter(Boolean); + const posthogIndex = segments.lastIndexOf(".posthog"); + if ( + posthogIndex < 0 || + segments[posthogIndex + 1] !== "attachments" || + !segments[posthogIndex + 2] || + !segments[posthogIndex + 3] + ) { + return undefined; + } + return { + runId: segments[posthogIndex + 2], + artifactId: segments[posthogIndex + 3], + }; +} + +function blockUri( + block: PromptContentBlock, +): { uri: string; name?: string } | null { + switch (block.type) { + case "resource": + return block.resource?.uri ? { uri: block.resource.uri } : null; + case "image": + return block.uri ? { uri: block.uri } : null; + case "resource_link": + return block.uri ? { uri: block.uri, name: block.name } : null; + default: + return null; + } +} + +function attachmentFromBlock( + block: PromptContentBlock, +): SessionNotificationAttachment | null { + const ref = blockUri(block); + if (!ref || !ref.uri.startsWith("file://")) return null; + + let pathname: string; + try { + pathname = decodeURIComponent(new URL(ref.uri).pathname); + } catch { + return null; + } + + const cloudArtifact = parseCloudArtifactRef(pathname); + if (!cloudArtifact) return null; + + const fileName = ref.name?.trim() || getFileName(pathname) || "attachment"; + return { + kind: isRasterImageFile(fileName) ? "image" : "document", + uri: ref.uri, + fileName, + cloudArtifact, + }; +} + +export function extractSessionPromptAttachments( + message: unknown, +): PromptAttachmentGroup | null { + const msg = message as PromptMessage | undefined; + if (msg?.method !== "session/prompt") return null; + const prompt = msg.params?.prompt; + if (!Array.isArray(prompt)) return null; + + const textParts: string[] = []; + const attachments: SessionNotificationAttachment[] = []; + for (const block of prompt) { + if (block.type === "text") { + if (block._meta?.ui?.hidden) continue; + if (typeof block.text === "string") textParts.push(block.text); + continue; + } + const attachment = attachmentFromBlock(block); + if (attachment) attachments.push(attachment); + } + + if (attachments.length === 0) return null; + return { text: textParts.join(""), attachments }; +} + +/** + * S3-backed snapshots replay user turns as text-only `user_message_chunk` + * events, dropping the attachment metadata. The `session/prompt` requests in the + * same log still carry the cloud artifact references, so reattach them by + * matching prompt text (FIFO on ties) to keep historical images renderable. + */ +export function reinjectPromptAttachments(events: SessionEvent[]): void { + const pending: PromptAttachmentGroup[] = []; + for (const event of events) { + if (event.type === "acp_message") { + const group = extractSessionPromptAttachments(event.message); + if (group) pending.push(group); + continue; + } + const update = event.notification?.update; + if (update?.sessionUpdate !== "user_message_chunk") continue; + if (update.attachments && update.attachments.length > 0) continue; + const text = update.content?.text ?? ""; + const idx = pending.findIndex((group) => group.text === text); + if (idx < 0) continue; + update.attachments = pending[idx].attachments; + pending.splice(idx, 1); + } +} diff --git a/apps/mobile/src/features/tasks/utils/sessionActivity.ts b/apps/mobile/src/features/tasks/utils/sessionActivity.ts index f980e66a52..982d4db8b6 100644 --- a/apps/mobile/src/features/tasks/utils/sessionActivity.ts +++ b/apps/mobile/src/features/tasks/utils/sessionActivity.ts @@ -1,11 +1,15 @@ -import type { SessionEvent, SessionNotification } from "../types"; +import type { + SessionEvent, + SessionNotification, + TerminalStatus, +} from "../types"; export type SessionActivityPhase = "idle" | "connecting" | "working"; interface SessionActivityState { isPromptPending?: boolean; awaitingAgentOutput?: boolean; - terminalStatus?: "failed" | "completed"; + terminalStatus?: TerminalStatus; events?: SessionEvent[]; }