diff --git a/src/agent-harness/image-highlights.test.ts b/src/agent-harness/image-highlights.test.ts new file mode 100644 index 0000000..631ccc1 --- /dev/null +++ b/src/agent-harness/image-highlights.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest" + +import { + mergeImageInspectionHighlights, + normalizeHighlightBoxes, + normalizeImageInspectionHighlights, +} from "./image-highlights" + +describe("image highlights", () => { + it("normalizes multi-page multi-region boxes and drops invalid refs", () => { + const highlights = normalizeImageInspectionHighlights({ + allowedRefs: new Set(["asset:page-1", "asset:page-2"]), + pages: [ + { + ref: "asset:page-1", + regions: [ + { x: 0.1, y: 0.2, w: 0.3, h: 0.1 }, + { x: 0.5, y: 0.6, w: 0.2, h: 0.15 }, + ], + }, + { + ref: "asset:page-2", + regions: [{ x: -0.1, y: 0.9, w: 0.3, h: 0.2 }], + }, + { + ref: "asset:unknown", + regions: [{ x: 0.1, y: 0.1, w: 0.2, h: 0.2 }], + }, + ], + }) + + expect(highlights).toEqual([ + { + ref: "asset:page-1", + regions: [ + { x: 0.1, y: 0.2, w: 0.3, h: 0.1 }, + { x: 0.5, y: 0.6, w: 0.2, h: 0.15 }, + ], + }, + { + ref: "asset:page-2", + regions: [{ x: 0, y: 0.9, w: 0.2, h: 0.1 }], + }, + ]) + }) + + it("clamps standalone highlight boxes for persistence reload", () => { + expect( + normalizeHighlightBoxes([ + { x: -0.2, y: 0.5, w: 0.4, h: 0.2 }, + { x: 0.1, y: 0.1, w: 0.001, h: 0.2 }, + ]), + ).toEqual([{ x: 0, y: 0.5, w: 0.2, h: 0.2 }]) + }) + + it("replaces prior regions for the same ref on later inspects", () => { + expect( + mergeImageInspectionHighlights( + [ + { + ref: "asset:page-1", + regions: [{ x: 0.1, y: 0.1, w: 0.2, h: 0.2 }], + }, + ], + [ + { + ref: "asset:page-1", + regions: [ + { x: 0.2, y: 0.3, w: 0.1, h: 0.1 }, + { x: 0.5, y: 0.5, w: 0.1, h: 0.1 }, + ], + }, + { + ref: "asset:page-3", + regions: [{ x: 0.1, y: 0.1, w: 0.2, h: 0.2 }], + }, + ], + ), + ).toEqual([ + { + ref: "asset:page-1", + regions: [ + { x: 0.2, y: 0.3, w: 0.1, h: 0.1 }, + { x: 0.5, y: 0.5, w: 0.1, h: 0.1 }, + ], + }, + { + ref: "asset:page-3", + regions: [{ x: 0.1, y: 0.1, w: 0.2, h: 0.2 }], + }, + ]) + }) +}) diff --git a/src/agent-harness/image-highlights.ts b/src/agent-harness/image-highlights.ts new file mode 100644 index 0000000..8adc037 --- /dev/null +++ b/src/agent-harness/image-highlights.ts @@ -0,0 +1,104 @@ +import type { ImageHighlightBox, ImageInspectionHighlights } from "./types" + +const MIN_BOX_SIZE = 0.01 + +export function normalizeImageInspectionHighlights(input: { + readonly pages: readonly { + readonly ref: string + readonly regions: readonly { + readonly x: number + readonly y: number + readonly w: number + readonly h: number + }[] + }[] + readonly allowedRefs: ReadonlySet +}): ImageInspectionHighlights[] { + const byRef = new Map() + + for (const page of input.pages) { + const ref = page.ref.trim() + if (!ref || !input.allowedRefs.has(ref)) continue + + const regions = normalizeHighlightBoxes(page.regions) + if (regions.length === 0) continue + + const existing = byRef.get(ref) ?? [] + byRef.set(ref, [...existing, ...regions]) + } + + return Array.from(byRef.entries()).map(([ref, regions]) => ({ + ref, + regions, + })) +} + +/** Clamp/normalize boxes for persistence and reload paths. */ +export function normalizeHighlightBoxes( + regions: readonly { + readonly x: number + readonly y: number + readonly w: number + readonly h: number + }[], +): ImageHighlightBox[] { + return regions + .map(clampHighlightBox) + .filter((box): box is ImageHighlightBox => box !== null) +} + +export function mergeImageInspectionHighlights( + existing: readonly ImageInspectionHighlights[] | undefined, + incoming: readonly ImageInspectionHighlights[] | undefined, +): ImageInspectionHighlights[] { + const byRef = new Map() + + for (const page of existing ?? []) { + byRef.set(page.ref, [...page.regions]) + } + for (const page of incoming ?? []) { + // Latest inspect for a ref replaces prior regions for that ref. + byRef.set(page.ref, [...page.regions]) + } + + return Array.from(byRef.entries()).map(([ref, regions]) => ({ + ref, + regions, + })) +} + +function clampHighlightBox(input: { + readonly x: number + readonly y: number + readonly w: number + readonly h: number +}): ImageHighlightBox | null { + if ( + !Number.isFinite(input.x) || + !Number.isFinite(input.y) || + !Number.isFinite(input.w) || + !Number.isFinite(input.h) + ) { + return null + } + + const x1 = clamp01(Math.min(input.x, input.x + input.w)) + const y1 = clamp01(Math.min(input.y, input.y + input.h)) + const x2 = clamp01(Math.max(input.x, input.x + input.w)) + const y2 = clamp01(Math.max(input.y, input.y + input.h)) + const w = x2 - x1 + const h = y2 - y1 + if (w < MIN_BOX_SIZE || h < MIN_BOX_SIZE) return null + + return { x: round01(x1), y: round01(y1), w: round01(w), h: round01(h) } +} + +function clamp01(value: number): number { + if (value <= 0) return 0 + if (value >= 1) return 1 + return value +} + +function round01(value: number): number { + return Math.round(value * 1_000_000) / 1_000_000 +} diff --git a/src/agent-harness/index.ts b/src/agent-harness/index.ts index 5fbdf7a..a2666bd 100644 --- a/src/agent-harness/index.ts +++ b/src/agent-harness/index.ts @@ -1,3 +1,4 @@ +export * from "./image-highlights" export * from "./ledger" export * from "./knowhere-text" export * from "./runtime" diff --git a/src/agent-harness/runtime.ts b/src/agent-harness/runtime.ts index 0ad95d8..3930146 100644 --- a/src/agent-harness/runtime.ts +++ b/src/agent-harness/runtime.ts @@ -10,6 +10,7 @@ import { z } from "zod" import { createEvidenceLedger } from "./ledger" import { knowhereToolText } from "./knowhere-text" +import { mergeImageInspectionHighlights } from "./image-highlights" import type { AgentTurn, AgentTurnInput, @@ -18,6 +19,7 @@ import type { HarnessToolCallTrace, HarnessTrace, ImageInspectionAsset, + ImageInspectionHighlights, ImageInspectionResponse, InspectImages, IntentFrame, @@ -50,6 +52,7 @@ type HarnessToolState = { finalized?: boolean priorTurnReads?: string[] inspectedImageRefs?: string[] + imageHighlights?: ImageInspectionHighlights[] toolCalls?: HarnessToolCallTrace[] } @@ -246,6 +249,7 @@ export async function runAgentHarness( finalized: state.finalized === true, priorTurnReads: [...(state.priorTurnReads ?? [])], toolCalls: [...(state.toolCalls ?? [])], + imageHighlights: [...(state.imageHighlights ?? [])], validationErrors: [], revisionsUsed: 0, }, @@ -761,8 +765,12 @@ async function inspectRetrievedImages(input: { question, assets: selectedAssets, }) + input.state.imageHighlights = mergeImageInspectionHighlights( + input.state.imageHighlights, + response.highlights, + ) return { - ok: true, + ok: true as const, analysis: response.analysis, inspected: response.inspected, skipped: [...skipped, ...response.skipped], diff --git a/src/agent-harness/types.ts b/src/agent-harness/types.ts index 36b6a27..edd9c2f 100644 --- a/src/agent-harness/types.ts +++ b/src/agent-harness/types.ts @@ -182,6 +182,23 @@ export type ImageInspectionInspectedAsset = { readonly label: string } +/** + * Normalized box relative to image width/height. Origin is top-left. + * Values are clamped to [0, 1] before render. + */ +export type ImageHighlightBox = { + readonly x: number + readonly y: number + readonly w: number + readonly h: number +} + +/** One page/image may contain multiple answer regions; no per-region labels. */ +export type ImageInspectionHighlights = { + readonly ref: string + readonly regions: readonly ImageHighlightBox[] +} + export type ImageInspectionRequest = { readonly question: string readonly assets: readonly ImageInspectionAsset[] @@ -191,6 +208,7 @@ export type ImageInspectionResponse = { readonly analysis: string readonly inspected: readonly ImageInspectionInspectedAsset[] readonly skipped: readonly ImageInspectionSkippedAsset[] + readonly highlights?: readonly ImageInspectionHighlights[] } export type InspectImages = ( @@ -256,6 +274,7 @@ export type HarnessTrace = { readonly finalized: boolean readonly priorTurnReads: readonly string[] readonly toolCalls: readonly HarnessToolCallTrace[] + readonly imageHighlights: readonly ImageInspectionHighlights[] readonly validationErrors: readonly string[] readonly revisionsUsed: number } diff --git a/src/components/chat-message-list.test.ts b/src/components/chat-message-list.test.ts index e59fd54..1f665fe 100644 --- a/src/components/chat-message-list.test.ts +++ b/src/components/chat-message-list.test.ts @@ -401,6 +401,62 @@ describe("ChatMessageList", () => { expect(screen.queryByRole("img", { name: "其他候选图片" })).toBeNull(); }); + it("renders multi-region answer highlights on displayed page artifacts", () => { + render( + React.createElement(ChatMessageList, { + messages: [ + { + id: "assistant_1", + role: "assistant", + content: "风险辨识要求建立分级管控制度。", + artifacts: [ + { + type: "image", + display: true, + assetUrl: "https://blob.example/pages/page-225.png", + label: "page 225", + highlightRegions: [ + { x: 0.1, y: 0.2, w: 0.4, h: 0.1 }, + { x: 0.2, y: 0.5, w: 0.5, h: 0.12 }, + ], + }, + ], + }, + ], + }), + ); + + expect(screen.getByTestId("chat-image-highlights")).toBeTruthy(); + expect(screen.getAllByTestId("chat-image-highlight-region")).toHaveLength(2); + }); + + it("keeps the original image layout when artifacts have no highlight regions", () => { + render( + React.createElement(ChatMessageList, { + messages: [ + { + id: "assistant_1", + role: "assistant", + content: "Here is the page.", + artifacts: [ + { + type: "image", + display: true, + assetUrl: "https://blob.example/pages/page-1.png", + label: "page 1", + }, + ], + }, + ], + }), + ); + + const image = screen.getByRole("img", { name: "page 1" }); + expect(image.className).toContain("object-contain"); + expect(image.className).toContain("w-full"); + expect(screen.queryByTestId("chat-image-highlights")).toBeNull(); + }); + it("does not fall back to image citations when a harness message has empty artifacts", () => { render( React.createElement(ChatMessageList, { diff --git a/src/components/chat-message-list.tsx b/src/components/chat-message-list.tsx index 64ab794..6007f94 100644 --- a/src/components/chat-message-list.tsx +++ b/src/components/chat-message-list.tsx @@ -35,12 +35,14 @@ type DisplayImageCitation = { readonly label: string; readonly tooltipLabel: string; readonly assetUrl: string; + readonly highlightRegions?: ChatArtifactView["highlightRegions"]; }; type DisplayImageArtifact = { readonly assetUrl: string; readonly citationId: string; readonly label: string; + readonly highlightRegions?: ChatArtifactView["highlightRegions"]; }; type DisplayDerivedTableArtifact = { @@ -349,16 +351,16 @@ function MessageBubble({ Images

- {displayImageCitations.map(({ assetUrl, citationId, label }) => ( + {displayImageCitations.map( + ({ assetUrl, citationId, label, highlightRegions }) => (
- {/* eslint-disable-next-line @next/next/no-img-element -- Chat image citation dimensions are not known before render. */} - {label}
@@ -366,7 +368,8 @@ function MessageBubble({
- ))} + ), + )}
)} @@ -809,12 +812,68 @@ function getDisplayImageArtifacts( assetUrl, citationId: `${message.id}:artifact:${index}`, label: getArtifactLabel(artifact, sourceTitlesByDocumentId), + ...(artifact.highlightRegions && artifact.highlightRegions.length > 0 + ? { highlightRegions: artifact.highlightRegions } + : {}), }); } return imageArtifacts; } +function HighlightedChatImage({ + assetUrl, + label, + highlightRegions, +}: { + readonly assetUrl: string; + readonly label: string; + readonly highlightRegions?: ChatArtifactView["highlightRegions"]; +}): ReactElement { + const regions = highlightRegions ?? []; + + if (regions.length === 0) { + return ( + // eslint-disable-next-line @next/next/no-img-element -- Chat image citation dimensions are not known before render. + {label} + ); + } + + return ( +
+ {/* eslint-disable-next-line @next/next/no-img-element -- Chat image citation dimensions are not known before render. */} + {label} + +
+ ); +} + function getDisplayDerivedTableArtifacts( message: ChatMessageView, ): readonly DisplayDerivedTableArtifact[] { diff --git a/src/domains/chat/chat-citation-persistence.ts b/src/domains/chat/chat-citation-persistence.ts index 5b0b26e..3753a9a 100644 --- a/src/domains/chat/chat-citation-persistence.ts +++ b/src/domains/chat/chat-citation-persistence.ts @@ -51,6 +51,7 @@ function toArtifactView(artifact: ChatArtifactView): ChatArtifactView { label: artifact.label, display: artifact.display, reason: artifact.reason, + highlightRegions: artifact.highlightRegions, citation: artifact.citation ? toCitationView(artifact.citation) : undefined, diff --git a/src/domains/chat/image-inspection-model.test.ts b/src/domains/chat/image-inspection-model.test.ts new file mode 100644 index 0000000..5e123a7 --- /dev/null +++ b/src/domains/chat/image-inspection-model.test.ts @@ -0,0 +1,298 @@ +import { + generateObject, + generateText, + NoObjectGeneratedError, + UnsupportedFunctionalityError, +} from "ai" +import { beforeEach, describe, expect, it, vi } from "vitest" + +import { + generateImageInspectionModelResult, + isRecoverableStructuredOutputError, + parseStructuredInspectionText, + salvageAnalysisFromFailedStructuredText, +} from "./image-inspection-model" + +vi.mock("ai", async (importOriginal) => { + const original = await importOriginal() + return { + ...original, + generateObject: vi.fn(), + generateText: vi.fn(), + } +}) + +describe("image inspection model", () => { + const assets = [ + { + ref: "asset:page-1", + label: "page 1", + body: new Uint8Array([1, 2, 3]), + contentType: "image/png", + }, + ] as const + + beforeEach(() => { + vi.mocked(generateObject).mockReset() + vi.mocked(generateText).mockReset() + }) + + it("returns structured analysis and pages on success", async () => { + vi.mocked(generateObject).mockResolvedValue({ + object: { + analysis: "The page states approval is required.", + pages: [ + { + ref: "asset:page-1", + regions: [{ x: 0.1, y: 0.2, w: 0.3, h: 0.4 }], + }, + ], + }, + } as Awaited>) + + const result = await generateImageInspectionModelResult({ + workspaceId: "ws_1", + question: "When is approval required?", + assets, + }) + + expect(result).toEqual({ + analysis: "The page states approval is required.", + pages: [ + { + ref: "asset:page-1", + regions: [{ x: 0.1, y: 0.2, w: 0.3, h: 0.4 }], + }, + ], + source: "structured", + }) + expect(generateText).not.toHaveBeenCalled() + }) + + it("salvages valid structured JSON from failed generateObject text", async () => { + vi.mocked(generateObject).mockRejectedValue( + makeNoObjectGeneratedError( + JSON.stringify({ + analysis: "Visible caption says 5000 yuan.", + pages: [ + { + ref: "asset:page-1", + regions: [{ x: 0.2, y: 0.3, w: 0.4, h: 0.1 }], + }, + ], + }), + ), + ) + + const result = await generateImageInspectionModelResult({ + workspaceId: "ws_1", + question: "What amount is shown?", + assets, + }) + + expect(result).toEqual({ + analysis: "Visible caption says 5000 yuan.", + pages: [ + { + ref: "asset:page-1", + regions: [{ x: 0.2, y: 0.3, w: 0.4, h: 0.1 }], + }, + ], + source: "structured_text", + }) + expect(generateText).not.toHaveBeenCalled() + }) + + it("salvages analysis-only when failed structured text has analysis but invalid pages", async () => { + vi.mocked(generateObject).mockRejectedValue( + makeNoObjectGeneratedError( + '{"analysis":"Visible caption says 5000 yuan.","pages":[{"ref":"bad"}]}', + ), + ) + + const result = await generateImageInspectionModelResult({ + workspaceId: "ws_1", + question: "What amount is shown?", + assets, + }) + + expect(result).toEqual({ + analysis: "Visible caption says 5000 yuan.", + pages: [], + source: "analysis_fallback", + }) + expect(generateText).not.toHaveBeenCalled() + }) + + it("falls back to generateText when structured output fails without salvageable analysis", async () => { + vi.mocked(generateObject).mockRejectedValue( + makeNoObjectGeneratedError('{"pages":[]}'), + ) + vi.mocked(generateText) + .mockResolvedValueOnce({ + text: "not-json", + } as Awaited>) + .mockResolvedValueOnce({ + text: "The image shows a fee table.", + } as Awaited>) + + const result = await generateImageInspectionModelResult({ + workspaceId: "ws_1", + question: "What does the table show?", + assets, + }) + + expect(result).toEqual({ + analysis: "The image shows a fee table.", + pages: [], + source: "analysis_fallback", + }) + expect(generateText).toHaveBeenCalledTimes(2) + }) + + it("uses generateText JSON when the model does not support structured object generation", async () => { + vi.mocked(generateObject).mockRejectedValue( + new UnsupportedFunctionalityError({ + functionality: "object generation", + }), + ) + vi.mocked(generateText).mockResolvedValue({ + text: JSON.stringify({ + analysis: "Approval is required on City land.", + pages: [ + { + ref: "asset:page-1", + regions: [{ x: 0.05, y: 0.1, w: 0.9, h: 0.2 }], + }, + ], + }), + } as Awaited>) + + const result = await generateImageInspectionModelResult({ + workspaceId: "ws_1", + question: "When is approval required?", + assets, + }) + + expect(result).toEqual({ + analysis: "Approval is required on City land.", + pages: [ + { + ref: "asset:page-1", + regions: [{ x: 0.05, y: 0.1, w: 0.9, h: 0.2 }], + }, + ], + source: "structured_text", + }) + expect(generateText).toHaveBeenCalledTimes(1) + }) + + it("falls back to analysis-only text when unsupported structured models cannot emit JSON", async () => { + vi.mocked(generateObject).mockRejectedValue( + new UnsupportedFunctionalityError({ + functionality: "object generation", + }), + ) + vi.mocked(generateText) + .mockResolvedValueOnce({ + text: "not-json", + } as Awaited>) + .mockResolvedValueOnce({ + text: "The diagram labels a 40km/h zone.", + } as Awaited>) + + const result = await generateImageInspectionModelResult({ + workspaceId: "ws_1", + question: "What speed is shown?", + assets, + }) + + expect(result).toEqual({ + analysis: "The diagram labels a 40km/h zone.", + pages: [], + source: "analysis_fallback", + }) + expect(generateText).toHaveBeenCalledTimes(2) + }) + + it("rethrows non-schema model failures without falling back", async () => { + vi.mocked(generateObject).mockRejectedValue(new Error("network down")) + + await expect( + generateImageInspectionModelResult({ + workspaceId: "ws_1", + question: "What does the table show?", + assets, + }), + ).rejects.toThrow("network down") + expect(generateText).not.toHaveBeenCalled() + }) + + it("classifies recoverable structured-output errors", () => { + expect( + isRecoverableStructuredOutputError( + new UnsupportedFunctionalityError({ + functionality: "object generation", + }), + ), + ).toBe(true) + expect( + isRecoverableStructuredOutputError(makeNoObjectGeneratedError("{}")), + ).toBe(true) + expect(isRecoverableStructuredOutputError(new Error("network down"))).toBe( + false, + ) + }) + + it("parses and salvages structured inspection text", () => { + expect( + parseStructuredInspectionText( + JSON.stringify({ + analysis: "Box A is labeled red.", + pages: [{ ref: "asset:page-1", regions: [{ x: 0, y: 0, w: 1, h: 1 }] }], + }), + ), + ).toEqual({ + analysis: "Box A is labeled red.", + pages: [{ ref: "asset:page-1", regions: [{ x: 0, y: 0, w: 1, h: 1 }] }], + }) + expect( + salvageAnalysisFromFailedStructuredText( + 'prefix {"analysis":"Box A is labeled red."} suffix', + ), + ).toBe("Box A is labeled red.") + expect(salvageAnalysisFromFailedStructuredText("Plain OCR notes.")).toBe( + "Plain OCR notes.", + ) + expect(salvageAnalysisFromFailedStructuredText('{"pages":[]}')).toBeNull() + expect(salvageAnalysisFromFailedStructuredText("")).toBeNull() + }) +}) + +function makeNoObjectGeneratedError(text: string): NoObjectGeneratedError { + return new NoObjectGeneratedError({ + message: "No object generated: response did not match schema.", + cause: new Error("schema mismatch"), + text, + response: { + id: "response_1", + modelId: "test-model", + timestamp: new Date("2026-01-01T00:00:00Z"), + }, + usage: { + inputTokens: 1, + inputTokenDetails: { + noCacheTokens: 1, + cacheReadTokens: 0, + cacheWriteTokens: 0, + }, + outputTokens: 1, + outputTokenDetails: { + textTokens: 1, + reasoningTokens: 0, + }, + totalTokens: 2, + }, + finishReason: "stop", + }) +} diff --git a/src/domains/chat/image-inspection-model.ts b/src/domains/chat/image-inspection-model.ts new file mode 100644 index 0000000..8589dda --- /dev/null +++ b/src/domains/chat/image-inspection-model.ts @@ -0,0 +1,388 @@ +import { + APICallError, + generateObject, + generateText, + NoObjectGeneratedError, + UnsupportedFunctionalityError, +} from "ai" +import { z } from "zod" + +import { CHAT_MODEL } from "@/lib/ai" +import { summarizeUnknownError } from "@/lib/format-log-value" +import { logger } from "@/lib/logger" + +const VISION_MODEL = process.env.VISION_MODEL ?? CHAT_MODEL + +export const imageInspectionResultSchema = z.object({ + analysis: z.string(), + pages: z + .array( + z.object({ + ref: z.string().min(1), + regions: z + .array( + z.object({ + x: z.number(), + y: z.number(), + w: z.number(), + h: z.number(), + }), + ) + .max(12), + }), + ) + .max(6) + .default([]), +}) + +export type ImageInspectionModelPage = z.infer< + typeof imageInspectionResultSchema +>["pages"][number] + +export type ImageInspectionModelAsset = { + readonly ref: string + readonly label: string + readonly body: Uint8Array + readonly contentType: string +} + +export type ImageInspectionModelResult = { + readonly analysis: string + readonly pages: readonly ImageInspectionModelPage[] + readonly source: "structured" | "structured_text" | "analysis_fallback" +} + +/** + * Prefer native structured output (analysis + highlight boxes). + * If the model/provider cannot do generateObject, degrade through: + * 1) salvage/parse JSON from failed text + * 2) generateText + JSON prompt (may still yield boxes) + * 3) analysis-only generateText + * so inspectImage still succeeds. + */ +export async function generateImageInspectionModelResult(input: { + readonly workspaceId: string + readonly question: string + readonly assets: readonly ImageInspectionModelAsset[] +}): Promise { + let structuredFailureText: string | undefined + + try { + const response = await generateObject({ + model: VISION_MODEL, + schema: imageInspectionResultSchema, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: buildImageInspectionStructuredPrompt({ + question: input.question, + assets: input.assets, + }), + }, + ...toImageParts(input.assets), + ], + }, + ], + }) + + return { + analysis: response.object.analysis.trim(), + pages: response.object.pages, + source: "structured", + } + } catch (error) { + if (!isRecoverableStructuredOutputError(error)) { + logger.warn("chat: image inspection model call failed", { + workspaceId: input.workspaceId, + model: VISION_MODEL, + inspectedCount: input.assets.length, + error: summarizeUnknownError(error), + }) + throw error + } + + structuredFailureText = getStructuredFailureText(error) + logger.warn("chat: image inspection structured output unavailable; falling back", { + workspaceId: input.workspaceId, + model: VISION_MODEL, + inspectedCount: input.assets.length, + generatedTextLength: structuredFailureText?.length ?? 0, + error: summarizeUnknownError(error), + }) + } + + const salvagedStructured = parseStructuredInspectionText(structuredFailureText) + if (salvagedStructured) { + return { + ...salvagedStructured, + source: "structured_text", + } + } + + const salvagedAnalysis = salvageAnalysisFromFailedStructuredText( + structuredFailureText, + ) + if (salvagedAnalysis !== null) { + return { + analysis: salvagedAnalysis, + pages: [], + source: "analysis_fallback", + } + } + + try { + const jsonTextResponse = await generateText({ + model: VISION_MODEL, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: buildImageInspectionStructuredPrompt({ + question: input.question, + assets: input.assets, + }), + }, + ...toImageParts(input.assets), + ], + }, + ], + experimental_include: { + requestBody: false, + responseBody: false, + }, + }) + + const parsedFromText = parseStructuredInspectionText(jsonTextResponse.text) + if (parsedFromText) { + return { + ...parsedFromText, + source: "structured_text", + } + } + + // Structured prompt asked for JSON; only keep an analysis field from JSON, + // never treat arbitrary freeform text as a successful inspection here. + const analysisFromJsonAttempt = salvageAnalysisFieldFromJsonText( + jsonTextResponse.text, + ) + if (analysisFromJsonAttempt !== null) { + return { + analysis: analysisFromJsonAttempt, + pages: [], + source: "analysis_fallback", + } + } + } catch (error) { + logger.warn("chat: image inspection structured-text fallback failed", { + workspaceId: input.workspaceId, + model: VISION_MODEL, + inspectedCount: input.assets.length, + error: summarizeUnknownError(error), + }) + } + + try { + const response = await generateText({ + model: VISION_MODEL, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: buildImageInspectionAnalysisPrompt({ + question: input.question, + assets: input.assets, + }), + }, + ...toImageParts(input.assets), + ], + }, + ], + experimental_include: { + requestBody: false, + responseBody: false, + }, + }) + + return { + analysis: response.text.trim(), + pages: [], + source: "analysis_fallback", + } + } catch (error) { + logger.warn("chat: image inspection analysis fallback failed", { + workspaceId: input.workspaceId, + model: VISION_MODEL, + inspectedCount: input.assets.length, + error: summarizeUnknownError(error), + }) + throw error + } +} + +export function isRecoverableStructuredOutputError(error: unknown): boolean { + if (NoObjectGeneratedError.isInstance(error)) return true + if (UnsupportedFunctionalityError.isInstance(error)) return true + + if (APICallError.isInstance(error)) { + const haystack = [ + error.message, + error.data === undefined ? "" : JSON.stringify(error.data), + typeof error.responseBody === "string" ? error.responseBody : "", + ] + .join(" ") + .toLowerCase() + + return ( + /response[_\s-]?format/.test(haystack) || + /json[_\s-]?schema/.test(haystack) || + /structured[_\s-]?output/.test(haystack) || + /tool[_\s-]?choice/.test(haystack) || + /unsupported/.test(haystack) + ) + } + + return false +} + +export function buildImageInspectionStructuredPrompt(input: { + readonly question: string + readonly assets: readonly Pick[] +}): string { + return [ + "Inspect the attached Notebook image assets selected from retrieved Knowhere evidence.", + "Answer the inspection question using concise visual observations only.", + "Use the provided refs to identify images. Do not include image URLs.", + "If OCR text is unclear, say it is unclear instead of guessing.", + "Do not create citations. The calling agent will cite the original retrieved asset refs.", + "", + "Also return answer provenance boxes for every page that supports the answer.", + "One answer may span multiple pages; each page may have one or more regions.", + "Coordinate system: origin top-left; x,y,w,h are fractions of image width/height in [0,1].", + "Do not add labels or captions for regions. Omit pages with no useful region.", + "", + "Return ONLY valid JSON with this shape:", + '{"analysis":"string","pages":[{"ref":"asset:page-12","regions":[{"x":0,"y":0,"w":0.2,"h":0.1}]}]}', + "", + "Inspection question:", + input.question, + "", + "Images:", + ...input.assets.map((asset) => `- ref=${asset.ref} label=${asset.label}`), + ].join("\n") +} + +export function buildImageInspectionAnalysisPrompt(input: { + readonly question: string + readonly assets: readonly Pick[] +}): string { + return [ + "Inspect the attached Notebook image assets selected from retrieved Knowhere evidence.", + "Answer the inspection question using concise visual observations only.", + "Use the provided refs and labels to identify images. Do not include image URLs.", + "If OCR text is unclear, say it is unclear instead of guessing.", + "Do not create citations. The calling agent will cite the original retrieved asset refs.", + "", + "Inspection question:", + input.question, + "", + "Images:", + ...input.assets.map((asset) => `- ref=${asset.ref} label=${asset.label}`), + ].join("\n") +} + +export function parseStructuredInspectionText( + text: string | undefined, +): { readonly analysis: string; readonly pages: ImageInspectionModelPage[] } | null { + const trimmed = text?.trim() + if (!trimmed) return null + + const start = trimmed.indexOf("{") + const end = trimmed.lastIndexOf("}") + if (start < 0 || end <= start) return null + + try { + const parsed = JSON.parse(trimmed.slice(start, end + 1)) as unknown + const result = imageInspectionResultSchema.safeParse(parsed) + if (!result.success) return null + + const analysis = result.data.analysis.trim() + if (!analysis) return null + + return { + analysis, + pages: result.data.pages, + } + } catch { + return null + } +} + +export function salvageAnalysisFieldFromJsonText( + text: string | undefined, +): string | null { + const trimmed = text?.trim() + if (!trimmed) return null + + try { + const start = trimmed.indexOf("{") + const end = trimmed.lastIndexOf("}") + if (start < 0 || end <= start) return null + + const parsed = JSON.parse(trimmed.slice(start, end + 1)) as unknown + if ( + !parsed || + typeof parsed !== "object" || + !("analysis" in parsed) || + typeof (parsed as { analysis: unknown }).analysis !== "string" + ) { + return null + } + + const analysis = (parsed as { analysis: string }).analysis.trim() + return analysis.length > 0 ? analysis : null + } catch { + return null + } +} + +export function salvageAnalysisFromFailedStructuredText( + text: string | undefined, +): string | null { + const structured = parseStructuredInspectionText(text) + if (structured) return structured.analysis + + const fromJson = salvageAnalysisFieldFromJsonText(text) + if (fromJson !== null) return fromJson + + const trimmed = text?.trim() + if (!trimmed) return null + + // Reject obvious raw JSON blobs without a usable analysis field. + if (trimmed.startsWith("{") || trimmed.startsWith("[")) { + return null + } + + return trimmed +} + +function getStructuredFailureText(error: unknown): string | undefined { + if (NoObjectGeneratedError.isInstance(error)) { + return error.text + } + return undefined +} + +function toImageParts(assets: readonly ImageInspectionModelAsset[]) { + return assets.map((asset) => ({ + type: "image" as const, + image: asset.body, + mediaType: asset.contentType, + })) +} diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index e26a670..114ade8 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -1545,6 +1545,7 @@ describe("answerQuestionWithRetrieval", () => { finalized: true, priorTurnReads: [], toolCalls: [], + imageHighlights: [], }, }; return harnessResult; @@ -2949,6 +2950,7 @@ function makeHarnessRunResult(text: string): HarnessRunResult { finalized: true, priorTurnReads: [], toolCalls: [], + imageHighlights: [], validationErrors: [], revisionsUsed: 0, }, diff --git a/src/domains/chat/index.ts b/src/domains/chat/index.ts index a6990c4..c4f7f85 100644 --- a/src/domains/chat/index.ts +++ b/src/domains/chat/index.ts @@ -9,6 +9,7 @@ import { logger } from "@/lib/logger" import type { ChatArtifactView, ChatCitationView, + ChatImageHighlightBox, } from "@/domains/chat/types" import type { DerivedTableArtifact, @@ -320,6 +321,11 @@ function toChatArtifactViewsFromHarness( chunk, ]), ) + const highlightsByRef = new Map( + (result.trace.imageHighlights ?? []).map( + (page) => [page.ref, page.regions] as const, + ), + ) const displayLimit = getHarnessArtifactDisplayLimit(result) const artifacts: ChatArtifactView[] = [] @@ -333,6 +339,7 @@ function toChatArtifactViewsFromHarness( artifact, assetsByRef, chunksByRef, + highlightsByRef, sources, }) if (!artifactView) continue @@ -381,6 +388,7 @@ function resolveHarnessArtifactView(input: { readonly artifact: OutputArtifact readonly assetsByRef: ReadonlyMap readonly chunksByRef: ReadonlyMap + readonly highlightsByRef: ReadonlyMap readonly sources: readonly AnswerQuestionInput["sources"][number][] }): ChatArtifactView | null { const asset = input.assetsByRef.get(input.artifact.ref) @@ -388,6 +396,7 @@ function resolveHarnessArtifactView(input: { return toChatArtifactView({ artifact: input.artifact, asset, + highlightRegions: input.highlightsByRef.get(asset.ref), sources: input.sources, }) } @@ -399,6 +408,9 @@ function resolveHarnessArtifactView(input: { ? toChatArtifactView({ artifact: input.artifact, asset: chunkAsset, + highlightRegions: + input.highlightsByRef.get(chunkAsset.ref) ?? + input.highlightsByRef.get(input.artifact.ref), sources: input.sources, }) : null @@ -407,6 +419,7 @@ function resolveHarnessArtifactView(input: { function toChatArtifactView(input: { readonly artifact: OutputArtifact readonly asset: EvidenceAsset + readonly highlightRegions?: readonly ChatImageHighlightBox[] readonly sources: readonly AnswerQuestionInput["sources"][number][] }): ChatArtifactView { const source = normalizeHarnessSource(input.asset.source, input.sources) @@ -417,6 +430,9 @@ function toChatArtifactView(input: { reason: input.artifact.reason, ...(input.asset.assetUrl ? { assetUrl: input.asset.assetUrl } : {}), label: input.asset.label, + ...(input.highlightRegions && input.highlightRegions.length > 0 + ? { highlightRegions: input.highlightRegions } + : {}), citation: { chunkType: input.asset.type, score: null, diff --git a/src/domains/chat/route-answer.ts b/src/domains/chat/route-answer.ts index ae8b13d..eaa8064 100644 --- a/src/domains/chat/route-answer.ts +++ b/src/domains/chat/route-answer.ts @@ -1,5 +1,4 @@ import { Cause, Effect, Either, Option } from "effect" -import { generateText } from "ai" import { generateAgenticOutputManifest, @@ -12,6 +11,8 @@ import type { ImageInspectionSkippedAsset, InspectImages, } from "@/agent-harness" +import { normalizeImageInspectionHighlights } from "@/agent-harness/image-highlights" +import { generateImageInspectionModelResult } from "@/domains/chat/image-inspection-model" import { hardenChatMediaAssetUrls } from "@/domains/chat/media-asset-hardening" import { handleChatTurn, @@ -256,15 +257,19 @@ async function inspectChatImages(input: { refs: preparedAssets.map((asset) => asset.ref), }) - const response = await generateImageInspectionText({ + const response = await generateImageInspectionModelResult({ workspaceId: input.workspaceId, question: input.request.question, assets: preparedAssets, }) const analysis = removeImageInspectionUrls({ - text: response.text.trim(), + text: response.analysis, assets: preparedAssets, }) + const highlights = normalizeImageInspectionHighlights({ + pages: response.pages, + allowedRefs: new Set(preparedAssets.map((asset) => asset.ref)), + }) logger.info("chat: image inspection response", { workspaceId: input.workspaceId, @@ -272,6 +277,12 @@ async function inspectChatImages(input: { inspectedCount: preparedAssets.length, skippedCount: skippedAssets.length, analysisLength: analysis.length, + inspectionSource: response.source, + highlightPageCount: highlights.length, + highlightRegionCount: highlights.reduce( + (count, page) => count + page.regions.length, + 0, + ), }) return { @@ -281,6 +292,7 @@ async function inspectChatImages(input: { label: asset.label, })), skipped: skippedAssets, + ...(highlights.length > 0 ? { highlights } : {}), } } @@ -382,49 +394,6 @@ async function prepareImageInspectionAsset(input: { } } -async function generateImageInspectionText(input: { - readonly workspaceId: string - readonly question: string - readonly assets: readonly PreparedImageInspectionAsset[] -}): Promise>> { - try { - return await generateText({ - model: VISION_MODEL, - messages: [ - { - role: "user", - content: [ - { - type: "text", - text: buildImageInspectionPrompt({ - question: input.question, - assets: input.assets, - }), - }, - ...input.assets.map((asset) => ({ - type: "image" as const, - image: asset.body, - mediaType: asset.contentType, - })), - ], - }, - ], - experimental_include: { - requestBody: false, - responseBody: false, - }, - }) - } catch (error) { - logger.warn("chat: image inspection model call failed", { - workspaceId: input.workspaceId, - model: VISION_MODEL, - inspectedCount: input.assets.length, - error: summarizeUnknownError(error), - }) - throw error - } -} - async function fetchPreparedInspectionImage(input: { readonly url: URL readonly contentType: string @@ -525,25 +494,6 @@ function getSupportedImageContentType(sourcePath: string): string | null { return extension ? SUPPORTED_IMAGE_CONTENT_TYPES[extension] ?? null : null } -function buildImageInspectionPrompt(input: { - readonly question: string - readonly assets: readonly PreparedImageInspectionAsset[] -}): string { - return [ - "Inspect the attached Notebook image assets selected from retrieved Knowhere evidence.", - "Answer the inspection question using concise visual observations only.", - "Use the provided refs and labels to identify images. Do not include image URLs.", - "If OCR text is unclear, say it is unclear instead of guessing.", - "Do not create citations. The calling agent will cite the original retrieved asset refs.", - "", - "Inspection question:", - input.question, - "", - "Images:", - ...input.assets.map((asset) => `- ref=${asset.ref} label=${asset.label}`), - ].join("\n") -} - function removeImageInspectionUrls(input: { readonly text: string readonly assets: readonly PreparedImageInspectionAsset[] diff --git a/src/domains/chat/route-service.test.ts b/src/domains/chat/route-service.test.ts index 97c9f6d..4eab67d 100644 --- a/src/domains/chat/route-service.test.ts +++ b/src/domains/chat/route-service.test.ts @@ -9,6 +9,7 @@ const mocks = vi.hoisted(() => ({ ensureDefaultChatThread: vi.fn(), findChatThreadInWorkspace: vi.fn(), generateAgenticOutputManifest: vi.fn(), + generateObject: vi.fn(), generateText: vi.fn(), getAuthenticated: vi.fn(), getAuthenticatedWithClient: vi.fn(), @@ -19,6 +20,7 @@ const mocks = vi.hoisted(() => ({ loggerInfo: vi.fn(), loggerWarn: vi.fn(), listSourcesForWorkspace: vi.fn(), + makeKnowhereClientWithParsedStorage: vi.fn(), parsedStorageGetAssetUrl: vi.fn(), parsedStorageWriteAsset: vi.fn(), softDeleteChatThread: vi.fn(), @@ -29,10 +31,15 @@ vi.mock("ai", async (importOriginal) => { const original = await importOriginal() return { ...original, + generateObject: mocks.generateObject, generateText: mocks.generateText, } }) +vi.mock("@/integrations/knowhere", () => ({ + makeKnowhereClientWithParsedStorage: mocks.makeKnowhereClientWithParsedStorage, +})) + vi.mock("@/domains/chat", async (importOriginal) => { const original = await importOriginal() return { @@ -99,6 +106,16 @@ describe("chat route services", () => { vi.clearAllMocks() mocks.parsedStorageGetAssetUrl.mockResolvedValue(null) mocks.parsedStorageWriteAsset.mockResolvedValue({ url: null }) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: {}, + knowledge: {}, + }) + mocks.generateObject.mockResolvedValue({ + object: { + analysis: "The image shows a chart.", + pages: [], + }, + }) mocks.generateText.mockResolvedValue({ text: "The image shows a chart." }) vi.stubGlobal( "fetch", @@ -253,8 +270,11 @@ describe("chat route services", () => { const durableUrl = "https://fake.public.blob.vercel-storage.com/workspaces/workspace_1/parsed-documents/doc_identity/job_1/images/id-front.png" mocks.parsedStorageWriteAsset.mockResolvedValue({ url: durableUrl }) - mocks.generateText.mockResolvedValue({ - text: `The card number is visible. ${durableUrl} ${rawUrl}`, + mocks.generateObject.mockResolvedValue({ + object: { + analysis: `The card number is visible. ${durableUrl} ${rawUrl}`, + pages: [], + }, }) mocks.getAuthenticatedWithClient.mockResolvedValue({ user: { id: "user_1" }, @@ -338,13 +358,9 @@ describe("chat route services", () => { body: new Uint8Array([1, 2, 3]), contentType: "image/png", }) - const generateInput = mocks.generateText.mock.calls[0]?.[0] + const generateInput = mocks.generateObject.mock.calls[0]?.[0] expect(generateInput).toMatchObject({ model: "google/gemini-3-flash", - experimental_include: { - requestBody: false, - responseBody: false, - }, }) expect(JSON.stringify(generateInput)).not.toContain( "knowhere-storage.example", @@ -368,8 +384,11 @@ describe("chat route services", () => { const durableUrl = "https://fake.public.blob.vercel-storage.com/workspaces/workspace_1/parsed-documents/doc_contract/job_1/page_citation_assets/page-8.png" mocks.parsedStorageGetAssetUrl.mockResolvedValue(durableUrl) - mocks.generateText.mockResolvedValue({ - text: "The page states 5000 yuan per occurrence.", + mocks.generateObject.mockResolvedValue({ + object: { + analysis: "The page states 5000 yuan per occurrence.", + pages: [], + }, }) mocks.getAuthenticatedWithClient.mockResolvedValue({ user: { id: "user_1" }, @@ -454,7 +473,7 @@ describe("chat route services", () => { }) expect(fetch).toHaveBeenCalledWith(durableUrl) expect(mocks.parsedStorageWriteAsset).not.toHaveBeenCalled() - const generateInput = mocks.generateText.mock.calls[0]?.[0] + const generateInput = mocks.generateObject.mock.calls[0]?.[0] const content = generateInput.messages[0].content const imagePart = content.find( (part: { readonly type: string }) => part.type === "image", @@ -471,8 +490,11 @@ describe("chat route services", () => { const durableUrl = "https://fake.public.blob.vercel-storage.com/workspaces/workspace_1/parsed-documents/doc_remote/job_remote/page_citation_assets/page-8.png" mocks.parsedStorageWriteAsset.mockResolvedValue({ url: durableUrl }) - mocks.generateText.mockResolvedValue({ - text: "The page states 5000 yuan per occurrence.", + mocks.generateObject.mockResolvedValue({ + object: { + analysis: "The page states 5000 yuan per occurrence.", + pages: [], + }, }) mocks.getAuthenticatedWithClient.mockResolvedValue({ user: { id: "user_1" }, @@ -567,7 +589,7 @@ describe("chat route services", () => { body: new Uint8Array([1, 2, 3]), contentType: "image/png", }) - const generateInput = mocks.generateText.mock.calls[0]?.[0] + const generateInput = mocks.generateObject.mock.calls[0]?.[0] expect(JSON.stringify(generateInput)).not.toContain( "knowhere-storage.example", ) @@ -674,6 +696,7 @@ describe("chat route services", () => { }) expect(result.status).toBe(200) + expect(mocks.generateObject).not.toHaveBeenCalled() expect(mocks.generateText).not.toHaveBeenCalled() expect(fetch).not.toHaveBeenCalled() }) diff --git a/src/domains/chat/service.test.ts b/src/domains/chat/service.test.ts index 6001171..b065251 100644 --- a/src/domains/chat/service.test.ts +++ b/src/domains/chat/service.test.ts @@ -350,6 +350,7 @@ function makeHarnessRunResult(text: string): HarnessRunResult { finalized: true, priorTurnReads: [], toolCalls: [], + imageHighlights: [], validationErrors: [], revisionsUsed: 0, }, diff --git a/src/domains/chat/types.ts b/src/domains/chat/types.ts index e430e4c..e5179b0 100644 --- a/src/domains/chat/types.ts +++ b/src/domains/chat/types.ts @@ -31,6 +31,17 @@ export type ChatCitationView = CitationView & { readonly content?: string } +/** + * Normalized highlight box for page/image answer provenance. + * Origin top-left; values in [0, 1]. No per-region labels. + */ +export type ChatImageHighlightBox = { + readonly x: number + readonly y: number + readonly w: number + readonly h: number +} + export type ChatArtifactView = { readonly type: "image" | "table" | "derived_table" readonly ref?: string @@ -42,6 +53,7 @@ export type ChatArtifactView = { readonly label?: string readonly display?: boolean readonly reason?: string + readonly highlightRegions?: readonly ChatImageHighlightBox[] readonly citation?: ChatCitationView } diff --git a/src/domains/chat/view.ts b/src/domains/chat/view.ts index f5c7f3a..f74c287 100644 --- a/src/domains/chat/view.ts +++ b/src/domains/chat/view.ts @@ -1,3 +1,4 @@ +import { normalizeHighlightBoxes } from "@/agent-harness/image-highlights" import { deriveChatThreadTitle } from "./title" import type { ChatMessage, ChatThread } from "@/infrastructure/db/schema" import type { @@ -137,6 +138,7 @@ function toPersistedArtifactViews(value: unknown): ChatArtifactView[] | undefine label: getString(item.label), display: typeof item.display === "boolean" ? item.display : undefined, reason: getString(item.reason), + highlightRegions: getHighlightRegions(item.highlightRegions), ...(citation ? { citation } : {}), }, ] @@ -145,6 +147,37 @@ function toPersistedArtifactViews(value: unknown): ChatArtifactView[] | undefine return artifacts.length > 0 ? artifacts : undefined } +function getHighlightRegions( + value: unknown, +): ChatArtifactView["highlightRegions"] { + if (!Array.isArray(value) || value.length === 0) return undefined + + const candidates = value.flatMap((item): Array<{ + x: number + y: number + w: number + h: number + }> => { + if (!isRecord(item)) return [] + const x = getNumber(item.x) + const y = getNumber(item.y) + const w = getNumber(item.w) + const h = getNumber(item.h) + if ( + x === undefined || + y === undefined || + w === undefined || + h === undefined + ) { + return [] + } + return [{ x, y, w, h }] + }) + + const regions = normalizeHighlightBoxes(candidates) + return regions.length > 0 ? regions : undefined +} + function getString(value: unknown): string | undefined { if (typeof value !== "string") return undefined return value.length > 0 ? value : undefined