feat: page answer provenance highlights on displayed images

src/agent-harness/image-highlights.test.ts

@@ -0,0 +1,93 @@
+import { describe, expect, it } from "vitest"
+
+import {
+  mergeImageInspectionHighlights,
+  normalizeHighlightBoxes,
+  normalizeImageInspectionHighlights,
+} from "./image-highlights"
+
+describe("image highlights", () => {
+  it("normalizes multi-page multi-region boxes and drops invalid refs", () => {
+    const highlights = normalizeImageInspectionHighlights({
+      allowedRefs: new Set(["asset:page-1", "asset:page-2"]),
+      pages: [
+        {
+          ref: "asset:page-1",
+          regions: [
+            { x: 0.1, y: 0.2, w: 0.3, h: 0.1 },
+            { x: 0.5, y: 0.6, w: 0.2, h: 0.15 },
+          ],
+        },
+        {
+          ref: "asset:page-2",
+          regions: [{ x: -0.1, y: 0.9, w: 0.3, h: 0.2 }],
+        },
+        {
+          ref: "asset:unknown",
+          regions: [{ x: 0.1, y: 0.1, w: 0.2, h: 0.2 }],
+        },
+      ],
+    })
+
+    expect(highlights).toEqual([
+      {
+        ref: "asset:page-1",
+        regions: [
+          { x: 0.1, y: 0.2, w: 0.3, h: 0.1 },
+          { x: 0.5, y: 0.6, w: 0.2, h: 0.15 },
+        ],
+      },
+      {
+        ref: "asset:page-2",
+        regions: [{ x: 0, y: 0.9, w: 0.2, h: 0.1 }],
+      },
+    ])
+  })
+
+  it("clamps standalone highlight boxes for persistence reload", () => {
+    expect(
+      normalizeHighlightBoxes([
+        { x: -0.2, y: 0.5, w: 0.4, h: 0.2 },
+        { x: 0.1, y: 0.1, w: 0.001, h: 0.2 },
+      ]),
+    ).toEqual([{ x: 0, y: 0.5, w: 0.2, h: 0.2 }])
+  })
+
+  it("replaces prior regions for the same ref on later inspects", () => {
+    expect(
+      mergeImageInspectionHighlights(
+        [
+          {
+            ref: "asset:page-1",
+            regions: [{ x: 0.1, y: 0.1, w: 0.2, h: 0.2 }],
+          },
+        ],
+        [
+          {
+            ref: "asset:page-1",
+            regions: [
+              { x: 0.2, y: 0.3, w: 0.1, h: 0.1 },
+              { x: 0.5, y: 0.5, w: 0.1, h: 0.1 },
+            ],
+          },
+          {
+            ref: "asset:page-3",
+            regions: [{ x: 0.1, y: 0.1, w: 0.2, h: 0.2 }],
+          },
+        ],
+      ),
+    ).toEqual([
+      {
+        ref: "asset:page-1",
+        regions: [
+          { x: 0.2, y: 0.3, w: 0.1, h: 0.1 },
+          { x: 0.5, y: 0.5, w: 0.1, h: 0.1 },
+        ],
+      },
+      {
+        ref: "asset:page-3",
+        regions: [{ x: 0.1, y: 0.1, w: 0.2, h: 0.2 }],
+      },
+    ])
+  })
+})

src/agent-harness/image-highlights.ts

@@ -0,0 +1,104 @@
+import type { ImageHighlightBox, ImageInspectionHighlights } from "./types"
+
+const MIN_BOX_SIZE = 0.01
+
+export function normalizeImageInspectionHighlights(input: {
+  readonly pages: readonly {
+    readonly ref: string
+    readonly regions: readonly {
+      readonly x: number
+      readonly y: number
+      readonly w: number
+      readonly h: number
+    }[]
+  }[]
+  readonly allowedRefs: ReadonlySet<string>
+}): ImageInspectionHighlights[] {
+  const byRef = new Map<string, ImageHighlightBox[]>()
+
+  for (const page of input.pages) {
+    const ref = page.ref.trim()
+    if (!ref || !input.allowedRefs.has(ref)) continue
+
+    const regions = normalizeHighlightBoxes(page.regions)
+    if (regions.length === 0) continue
+
+    const existing = byRef.get(ref) ?? []
+    byRef.set(ref, [...existing, ...regions])
+  }
+
+  return Array.from(byRef.entries()).map(([ref, regions]) => ({
+    ref,
+    regions,
+  }))
+}
+
+/** Clamp/normalize boxes for persistence and reload paths. */
+export function normalizeHighlightBoxes(
+  regions: readonly {
+    readonly x: number
+    readonly y: number
+    readonly w: number
+    readonly h: number
+  }[],
+): ImageHighlightBox[] {
+  return regions
+    .map(clampHighlightBox)
+    .filter((box): box is ImageHighlightBox => box !== null)
+}
+
+export function mergeImageInspectionHighlights(
+  existing: readonly ImageInspectionHighlights[] | undefined,
+  incoming: readonly ImageInspectionHighlights[] | undefined,
+): ImageInspectionHighlights[] {
+  const byRef = new Map<string, ImageHighlightBox[]>()
+
+  for (const page of existing ?? []) {
+    byRef.set(page.ref, [...page.regions])
+  }
+  for (const page of incoming ?? []) {
+    // Latest inspect for a ref replaces prior regions for that ref.
+    byRef.set(page.ref, [...page.regions])
+  }
+
+  return Array.from(byRef.entries()).map(([ref, regions]) => ({
+    ref,
+    regions,
+  }))
+}
+
+function clampHighlightBox(input: {
+  readonly x: number
+  readonly y: number
+  readonly w: number
+  readonly h: number
+}): ImageHighlightBox | null {
+  if (
+    !Number.isFinite(input.x) ||
+    !Number.isFinite(input.y) ||
+    !Number.isFinite(input.w) ||
+    !Number.isFinite(input.h)
+  ) {
+    return null
+  }
+
+  const x1 = clamp01(Math.min(input.x, input.x + input.w))
+  const y1 = clamp01(Math.min(input.y, input.y + input.h))
+  const x2 = clamp01(Math.max(input.x, input.x + input.w))
+  const y2 = clamp01(Math.max(input.y, input.y + input.h))
+  const w = x2 - x1
+  const h = y2 - y1
+  if (w < MIN_BOX_SIZE || h < MIN_BOX_SIZE) return null
+
+  return { x: round01(x1), y: round01(y1), w: round01(w), h: round01(h) }
+}
+
+function clamp01(value: number): number {
+  if (value <= 0) return 0
+  if (value >= 1) return 1
+  return value
+}
+
+function round01(value: number): number {
+  return Math.round(value * 1_000_000) / 1_000_000
+}

src/agent-harness/index.ts

@@ -1,3 +1,4 @@
+export * from "./image-highlights"
 export * from "./ledger"
 export * from "./knowhere-text"
 export * from "./runtime"

src/agent-harness/runtime.ts

@@ -10,6 +10,7 @@ import { z } from "zod"
 
 import { createEvidenceLedger } from "./ledger"
 import { knowhereToolText } from "./knowhere-text"
+import { mergeImageInspectionHighlights } from "./image-highlights"
 import type {
   AgentTurn,
   AgentTurnInput,
@@ -18,6 +19,7 @@ import type {
   HarnessToolCallTrace,
   HarnessTrace,
   ImageInspectionAsset,
+  ImageInspectionHighlights,
   ImageInspectionResponse,
   InspectImages,
   IntentFrame,
@@ -50,6 +52,7 @@ type HarnessToolState = {
   finalized?: boolean
   priorTurnReads?: string[]
   inspectedImageRefs?: string[]
+  imageHighlights?: ImageInspectionHighlights[]
   toolCalls?: HarnessToolCallTrace[]
 }
 
@@ -246,6 +249,7 @@ export async function runAgentHarness(
       finalized: state.finalized === true,
       priorTurnReads: [...(state.priorTurnReads ?? [])],
       toolCalls: [...(state.toolCalls ?? [])],
+      imageHighlights: [...(state.imageHighlights ?? [])],
       validationErrors: [],
       revisionsUsed: 0,
     },
@@ -761,8 +765,12 @@ async function inspectRetrievedImages(input: {
       question,
       assets: selectedAssets,
     })
+    input.state.imageHighlights = mergeImageInspectionHighlights(
+      input.state.imageHighlights,
+      response.highlights,
+    )
     return {
-      ok: true,
+      ok: true as const,
       analysis: response.analysis,
       inspected: response.inspected,
       skipped: [...skipped, ...response.skipped],

src/agent-harness/types.ts

@@ -182,6 +182,23 @@ export type ImageInspectionInspectedAsset = {
   readonly label: string
 }
 
+/**
+ * Normalized box relative to image width/height. Origin is top-left.
+ * Values are clamped to [0, 1] before render.
+ */
+export type ImageHighlightBox = {
+  readonly x: number
+  readonly y: number
+  readonly w: number
+  readonly h: number
+}
+
+/** One page/image may contain multiple answer regions; no per-region labels. */
+export type ImageInspectionHighlights = {
+  readonly ref: string
+  readonly regions: readonly ImageHighlightBox[]
+}
+
 export type ImageInspectionRequest = {
   readonly question: string
   readonly assets: readonly ImageInspectionAsset[]
@@ -191,6 +208,7 @@ export type ImageInspectionResponse = {
   readonly analysis: string
   readonly inspected: readonly ImageInspectionInspectedAsset[]
   readonly skipped: readonly ImageInspectionSkippedAsset[]
+  readonly highlights?: readonly ImageInspectionHighlights[]
 }
 
 export type InspectImages = (
@@ -256,6 +274,7 @@ export type HarnessTrace = {
   readonly finalized: boolean
   readonly priorTurnReads: readonly string[]
   readonly toolCalls: readonly HarnessToolCallTrace[]
+  readonly imageHighlights: readonly ImageInspectionHighlights[]
   readonly validationErrors: readonly string[]
   readonly revisionsUsed: number
 }

src/components/chat-message-list.test.ts

@@ -401,6 +401,62 @@ describe("ChatMessageList", () => {
     expect(screen.queryByRole("img", { name: "其他候选图片" })).toBeNull();
   });
 
+  it("renders multi-region answer highlights on displayed page artifacts", () => {
+    render(
+      React.createElement(ChatMessageList, {
+        messages: [
+          {
+            id: "assistant_1",
+            role: "assistant",
+            content: "风险辨识要求建立分级管控制度。",
+            artifacts: [
+              {
+                type: "image",
+                display: true,
+                assetUrl: "https://blob.example/pages/page-225.png",
+                label: "page 225",
+                highlightRegions: [
+                  { x: 0.1, y: 0.2, w: 0.4, h: 0.1 },
+                  { x: 0.2, y: 0.5, w: 0.5, h: 0.12 },
+                ],
+              },
+            ],
+          },
+        ],
+      }),
+    );
+
+    expect(screen.getByTestId("chat-image-highlights")).toBeTruthy();
+    expect(screen.getAllByTestId("chat-image-highlight-region")).toHaveLength(2);
+  });
+
+  it("keeps the original image layout when artifacts have no highlight regions", () => {
+    render(
+      React.createElement(ChatMessageList, {
+        messages: [
+          {
+            id: "assistant_1",
+            role: "assistant",
+            content: "Here is the page.",
+            artifacts: [
+              {
+                type: "image",
+                display: true,
+                assetUrl: "https://blob.example/pages/page-1.png",
+                label: "page 1",
+              },
+            ],
+          },
+        ],
+      }),
+    );
+
+    const image = screen.getByRole("img", { name: "page 1" });
+    expect(image.className).toContain("object-contain");
+    expect(image.className).toContain("w-full");
+    expect(screen.queryByTestId("chat-image-highlights")).toBeNull();
+  });
+
   it("does not fall back to image citations when a harness message has empty artifacts", () => {
     render(
       React.createElement(ChatMessageList, {