From 1b9d082266f1560cdda4aa2a426022696ef36eaa Mon Sep 17 00:00:00 2001 From: suguanYang Date: Mon, 10 Aug 2026 00:35:43 +0800 Subject: [PATCH 01/11] feat(chat): add agentic retrieval toggle to chat composer --- src/components/chat-composer.test.ts | 43 +++++++++- src/components/chat-composer.tsx | 79 ++++++++++++++----- src/components/chat-panel.test.ts | 4 +- src/components/chat-panel.tsx | 14 +++- .../workspace-chat-workflow.test.ts | 13 ++- src/components/workspace-chat-workflow.ts | 12 ++- src/components/workspace-shell-layout.tsx | 6 +- src/domains/chat/contracts.ts | 1 + src/domains/chat/index.test.ts | 17 ++++ src/domains/chat/index.ts | 5 +- src/domains/chat/request.ts | 3 + src/domains/chat/route-answer.ts | 1 + src/domains/chat/route-service.test.ts | 2 + src/domains/chat/service.ts | 2 + src/domains/workspace/client.ts | 1 + 15 files changed, 171 insertions(+), 32 deletions(-) diff --git a/src/components/chat-composer.test.ts b/src/components/chat-composer.test.ts index 1d7051b1..0853e00e 100644 --- a/src/components/chat-composer.test.ts +++ b/src/components/chat-composer.test.ts @@ -28,10 +28,51 @@ describe("ChatComposer", () => { await user.type(input, " Summarize this document "); await user.click(screen.getByRole("button", { name: "Send message" })); - expect(onSend).toHaveBeenCalledWith("Summarize this document"); + expect(onSend).toHaveBeenCalledWith("Summarize this document", { + useAgentic: true, + }); expect(input.value).toBe(""); }); + it("defaults to agentic retrieval enabled and explains the toggle", async () => { + const user = userEvent.setup(); + + render(React.createElement(ChatComposer)); + + const toggle = screen.getByRole("button", { + name: "Toggle agentic retrieval", + }); + expect(toggle.getAttribute("aria-pressed")).toBe("true"); + + await user.hover(toggle); + + const tooltip = await screen.findByRole("tooltip"); + expect(tooltip.textContent).toContain( + "Agentic retrieval plans document selection and navigation", + ); + }); + + it("sends useAgentic false after toggling agentic retrieval off", async () => { + const user = userEvent.setup(); + const onSend = vi.fn(); + + render(React.createElement(ChatComposer, { onSend })); + + const toggle = screen.getByRole("button", { + name: "Toggle agentic retrieval", + }); + await user.click(toggle); + expect(toggle.getAttribute("aria-pressed")).toBe("false"); + + const input = getComposerTextArea(); + await user.type(input, "Quick summary"); + await user.click(screen.getByRole("button", { name: "Send message" })); + + expect(onSend).toHaveBeenCalledWith("Quick summary", { + useAgentic: false, + }); + }); + it("caps long prompts and resets the composer after sending", async () => { const user = userEvent.setup(); const onSend = vi.fn(); diff --git a/src/components/chat-composer.tsx b/src/components/chat-composer.tsx index 4cfa62c2..e9ee0859 100644 --- a/src/components/chat-composer.tsx +++ b/src/components/chat-composer.tsx @@ -10,7 +10,7 @@ import { type MouseEvent, type ReactElement, } from "react"; -import { BarChart3, FileText, Plus, Send } from "lucide-react"; +import { BarChart3, FileText, Plus, Send, Sparkles } from "lucide-react"; import { Button } from "@/components/ui/button"; import { @@ -22,6 +22,12 @@ import { } from "@/components/ui/dropdown-menu"; import { Spinner } from "@/components/ui/spinner"; import { Textarea } from "@/components/ui/textarea"; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from "@/components/ui/tooltip"; import { chatPromptTemplates } from "@/domains/chat/prompt-templates"; const chatComposerName = "chat-composer"; const chatComposerTextAreaMinHeight = 128; @@ -33,6 +39,10 @@ type TextRange = { readonly end: number; }; +export type ChatSendOptions = { + readonly useAgentic: boolean; +}; + export type ChatComposerProps = { readonly canCreateDiagram?: boolean; readonly isDisabled?: boolean; @@ -40,7 +50,7 @@ export type ChatComposerProps = { readonly isSending?: boolean; readonly onCreateDiagram?: () => void; readonly onLoginClick?: () => void; - readonly onSend?: (text: string) => void; + readonly onSend?: (text: string, options: ChatSendOptions) => void; }; export function ChatComposer({ @@ -53,6 +63,7 @@ export function ChatComposer({ onSend, }: ChatComposerProps): ReactElement { const [input, setInput] = useState(""); + const [useAgentic, setUseAgentic] = useState(true); const composerInputId = useId(); const pendingTemplatePromptRef = useRef(null); const textareaRef = useRef(null); @@ -79,7 +90,7 @@ export function ChatComposer({ function handleSend(): void { if (!canSend) return; - onSend?.(trimmedInput); + onSend?.(trimmedInput, { useAgentic }); setInput(""); } @@ -177,22 +188,52 @@ export function ChatComposer({ onCreateDiagram={onCreateDiagram} onTemplateSelect={handleTemplateSelect} /> - +
+ + + + + + + Agentic retrieval plans document selection and navigation + for more thorough answers. Turn off for faster classic + search. + + + + +
)} diff --git a/src/components/chat-panel.test.ts b/src/components/chat-panel.test.ts index 54ccce58..f1bd6d1d 100644 --- a/src/components/chat-panel.test.ts +++ b/src/components/chat-panel.test.ts @@ -254,7 +254,9 @@ describe("ChatPanel", () => { ); await user.click(screen.getByRole("button", { name: "Send message" })); - expect(onSend).toHaveBeenCalledWith("Summarize revenue"); + expect(onSend).toHaveBeenCalledWith("Summarize revenue", { + useAgentic: true, + }); expect( analyticsMocks.trackNotebookAssistantQuestionSubmitted, ).toHaveBeenCalledWith({ diff --git a/src/components/chat-panel.tsx b/src/components/chat-panel.tsx index 92d3cbfa..1f0d1e87 100644 --- a/src/components/chat-panel.tsx +++ b/src/components/chat-panel.tsx @@ -6,7 +6,10 @@ import { type ReactElement, } from "react"; import { History, Plus } from "lucide-react"; -import { ChatComposer } from "@/components/chat-composer"; +import { + ChatComposer, + type ChatSendOptions, +} from "@/components/chat-composer"; import { ChatHistorySheet } from "@/components/chat-history-sheet"; import { ChatMessageList, @@ -47,7 +50,7 @@ export type ChatPanelProps = { messages: ChatMessageView[]; threads: ChatThreadView[]; activeThreadId?: string | null; - onSend?: (text: string) => void; + onSend?: (text: string, options: ChatSendOptions) => void; onNewChat?: () => void; onThreadSelect?: (threadId: string) => void; onThreadArchive?: (threadId: string) => void; @@ -152,7 +155,10 @@ export function ChatPanel({ } } - function handleComposerSend(text: string): void { + function handleComposerSend( + text: string, + options: ChatSendOptions, + ): void { if (isCreateDiagramCommand(text)) { void handleCreateDiagramCommand(); return; @@ -165,7 +171,7 @@ export function ChatPanel({ sourceCountSnapshot: sourceCount, messageLength: text.length, }); - onSend?.(text); + onSend?.(text, options); } return ( diff --git a/src/components/workspace-chat-workflow.test.ts b/src/components/workspace-chat-workflow.test.ts index f6b36a3f..babf01a9 100644 --- a/src/components/workspace-chat-workflow.test.ts +++ b/src/components/workspace-chat-workflow.test.ts @@ -84,12 +84,15 @@ describe("useWorkspaceChatWorkflow", () => { }) await act(async () => { - await result.current.handleChatSend("Summarize it") + await result.current.handleChatSend("Summarize it", { + useAgentic: true, + }) }) expect(mocks.sendChatMessage).toHaveBeenCalledWith({ message: "Summarize it", threadId: undefined, + useAgentic: true, excludedSourceIds: ["source_excluded"], }) await waitFor(() => { @@ -127,7 +130,9 @@ describe("useWorkspaceChatWorkflow", () => { }) await act(async () => { - await result.current.handleChatSend("What changed in Q4?") + await result.current.handleChatSend("What changed in Q4?", { + useAgentic: true, + }) }) expect(mocks.materializeDemoSources).toHaveBeenCalledWith({ @@ -172,7 +177,9 @@ describe("useWorkspaceChatWorkflow", () => { }) await act(async () => { - await result.current.handleChatSend("Summarize it") + await result.current.handleChatSend("Summarize it", { + useAgentic: true, + }) }) expect(mocks.materializeDemoSources).not.toHaveBeenCalled() diff --git a/src/components/workspace-chat-workflow.ts b/src/components/workspace-chat-workflow.ts index 2943a236..dd855763 100644 --- a/src/components/workspace-chat-workflow.ts +++ b/src/components/workspace-chat-workflow.ts @@ -12,6 +12,7 @@ import { type AnalyticsContext, } from "@/lib/posthog" import { workspaceClient } from "@/domains/workspace/client" +import type { ChatSendOptions } from "@/components/chat-composer" import { workspaceClientCache, type ChatThreadDetailResponse, @@ -43,7 +44,10 @@ type WorkspaceChatWorkflow = { readonly chat: ReturnType readonly chatThreads: ChatThreadView[] readonly handleArchiveChatThread: (threadId: string) => Promise - readonly handleChatSend: (text: string) => Promise + readonly handleChatSend: ( + text: string, + options: ChatSendOptions, + ) => Promise readonly handleCreateChatThread: () => Promise readonly handleRefreshActiveChatThread: () => Promise readonly handleSelectChatThread: (threadId: string) => void @@ -247,7 +251,10 @@ export function useWorkspaceChatWorkflow({ } } - async function handleChatSend(text: string): Promise { + async function handleChatSend( + text: string, + options: ChatSendOptions, + ): Promise { const sendStart = Date.now() const selectedSourcesCount = sources.filter( (source) => @@ -297,6 +304,7 @@ export function useWorkspaceChatWorkflow({ const body = await sendChatMessage({ message: text, threadId: chat.threadId ?? undefined, + useAgentic: options.useAgentic, excludedSourceIds: sources .filter((source) => source.excludedFromQuery) .map((source) => source.id), diff --git a/src/components/workspace-shell-layout.tsx b/src/components/workspace-shell-layout.tsx index 235eb075..760aa075 100644 --- a/src/components/workspace-shell-layout.tsx +++ b/src/components/workspace-shell-layout.tsx @@ -8,6 +8,7 @@ import { } from "lucide-react" import { ChatPanel } from "@/components/chat-panel" +import type { ChatSendOptions } from "@/components/chat-composer" import { ChunksPanel } from "@/components/chunks-panel" import { MobileTabBar } from "@/components/mobile-tab-bar" import { OfficialLibraryPanel } from "@/components/official-library-panel" @@ -91,7 +92,10 @@ export type WorkspaceShellLayoutProps = { readonly onArchiveChatThread: (threadId: string) => void | Promise readonly onArchiveSource: (sourceId: string) => void | Promise readonly onRetrySource?: (sourceId: string) => void | Promise - readonly onChatSend: (text: string) => void | Promise + readonly onChatSend: ( + text: string, + options: ChatSendOptions, + ) => void | Promise readonly onCitationClick: ( citation: ChatCitationView, citationId: string, diff --git a/src/domains/chat/contracts.ts b/src/domains/chat/contracts.ts index 4c8d224d..c7820b3b 100644 --- a/src/domains/chat/contracts.ts +++ b/src/domains/chat/contracts.ts @@ -65,6 +65,7 @@ export type AnswerQuestionInput = { namespaces?: readonly string[] sources: readonly Source[] excludedSourceIds: readonly string[] + useAgentic?: boolean retrieval: RetrievalClient generateAnswer: GenerateAnswer loadSourceAssetUrls?: LoadSourceAssetUrls diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index 0c8cfd65..9ddfb640 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -1734,11 +1734,28 @@ describe("parseChatRequestBody", () => { value: { question: "What changed?", threadId: "thread_1", + useAgentic: true, excludedSourceIds: ["source_1", "source_2"], }, }); }); + it("keeps an explicit useAgentic choice from the request body", () => { + expect( + parseChatRequestBody({ + message: "Quick summary", + useAgentic: false, + }), + ).toEqual({ + ok: true, + value: { + question: "Quick summary", + useAgentic: false, + excludedSourceIds: [], + }, + }); + }); + it("rejects empty questions before retrieval or model calls", () => { expect(parseChatRequestBody({ message: " " })).toEqual({ ok: false, diff --git a/src/domains/chat/index.ts b/src/domains/chat/index.ts index 2d7245d1..2fd3b172 100644 --- a/src/domains/chat/index.ts +++ b/src/domains/chat/index.ts @@ -132,6 +132,7 @@ export const answerQuestionWithRetrieval = ( input: queryInput, fallbackQuestion: question, namespace, + useAgentic: input.useAgentic ?? true, sources: input.sources, excludedSourceIds: input.excludedSourceIds, }) @@ -139,6 +140,7 @@ export const answerQuestionWithRetrieval = ( namespace, query: retrievalQueryParams.query, topK: retrievalQueryParams.topK, + useAgentic: retrievalQueryParams.useAgentic, dataType: retrievalQueryParams.dataType ?? null, signalPathCount: retrievalQueryParams.signalPaths?.length ?? 0, filterMode: retrievalQueryParams.filterMode ?? null, @@ -686,6 +688,7 @@ function buildRetrievalQueryParams(input: { readonly input: AgenticRetrievalQuery readonly fallbackQuestion: string readonly namespace: string + readonly useAgentic: boolean readonly sources: AnswerQuestionInput["sources"] readonly excludedSourceIds: readonly string[] }): RetrievalQueryParams { @@ -698,7 +701,7 @@ function buildRetrievalQueryParams(input: { namespace: input.namespace, query, topK: normalizeTopK(input.input.topK), - useAgentic: true, + useAgentic: input.useAgentic, dataType, ...(input.input.signalPaths && input.input.signalPaths.length > 0 ? { signalPaths: input.input.signalPaths } diff --git a/src/domains/chat/request.ts b/src/domains/chat/request.ts index c15a76ea..3e56f064 100644 --- a/src/domains/chat/request.ts +++ b/src/domains/chat/request.ts @@ -3,6 +3,7 @@ import { Either, Schema } from "effect" export type ParsedChatRequest = { question: string threadId?: string + useAgentic: boolean excludedSourceIds: string[] } @@ -13,6 +14,7 @@ export type ParseChatRequestResult = const ChatRequestBody = Schema.Struct({ message: Schema.String, threadId: Schema.optional(Schema.String), + useAgentic: Schema.optional(Schema.Boolean), excludedSourceIds: Schema.optional(Schema.Array(Schema.Unknown)), }) @@ -43,6 +45,7 @@ export function parseChatRequestBody(body: unknown): ParseChatRequestResult { parsed.threadId !== undefined && parsed.threadId.length > 0 ? parsed.threadId : undefined, + useAgentic: parsed.useAgentic ?? true, excludedSourceIds, }, } diff --git a/src/domains/chat/route-answer.ts b/src/domains/chat/route-answer.ts index ea759438..1d792c53 100644 --- a/src/domains/chat/route-answer.ts +++ b/src/domains/chat/route-answer.ts @@ -78,6 +78,7 @@ const answerChatEffect = (input: AnswerChatInput) => sources, question: body.value.question, threadId: body.value.threadId, + useAgentic: body.value.useAgentic, excludedSourceIds: body.value.excludedSourceIds, retrieval: client.retrieval, generateAnswer: generateAgenticOutputManifest, diff --git a/src/domains/chat/route-service.test.ts b/src/domains/chat/route-service.test.ts index 4358e8a1..3b1d7895 100644 --- a/src/domains/chat/route-service.test.ts +++ b/src/domains/chat/route-service.test.ts @@ -104,6 +104,7 @@ describe("chat route services", () => { body: { message: " Summarize it ", threadId: "thread_1", + useAgentic: true, excludedSourceIds: ["source_skipped", null], }, }) @@ -126,6 +127,7 @@ describe("chat route services", () => { sources: [readySource], question: "Summarize it", threadId: "thread_1", + useAgentic: true, excludedSourceIds: ["source_skipped"], retrieval: client.retrieval, generateAnswer: mocks.generateAgenticOutputManifest, diff --git a/src/domains/chat/service.ts b/src/domains/chat/service.ts index 5361d5b4..9691ddf7 100644 --- a/src/domains/chat/service.ts +++ b/src/domains/chat/service.ts @@ -64,6 +64,7 @@ type ChatTurnInput = { sources: readonly Source[] question: string threadId?: string + useAgentic?: boolean excludedSourceIds: readonly string[] retrieval: RetrievalClient generateAnswer: GenerateAnswer @@ -123,6 +124,7 @@ export const handleChatTurnEffect = (input: ChatTurnInput) => namespace: input.workspace.namespace, namespaces: getCompatibleNamespaces(input.workspace), sources: readySources, + useAgentic: input.useAgentic ?? true, excludedSourceIds: input.excludedSourceIds, retrieval: input.retrieval, generateAnswer: input.generateAnswer, diff --git a/src/domains/workspace/client.ts b/src/domains/workspace/client.ts index 9a7b3cd1..d57eebce 100644 --- a/src/domains/workspace/client.ts +++ b/src/domains/workspace/client.ts @@ -45,6 +45,7 @@ type ChatThreadDetailResponse = ChatThreadResponse & { type ChatMessageRequest = { message: string threadId?: string + useAgentic: boolean excludedSourceIds: string[] } From 69edcb4f9e7580285e3a93a7a2c7d0f240a49aaf Mon Sep 17 00:00:00 2001 From: suguanYang Date: Mon, 10 Aug 2026 09:37:35 +0800 Subject: [PATCH 02/11] feat(chat): refine deep search toggle UI --- src/components/chat-composer.test.ts | 18 ++++++------- src/components/chat-composer.tsx | 40 +++++++++++++++------------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/src/components/chat-composer.test.ts b/src/components/chat-composer.test.ts index 0853e00e..a12264a2 100644 --- a/src/components/chat-composer.test.ts +++ b/src/components/chat-composer.test.ts @@ -34,35 +34,35 @@ describe("ChatComposer", () => { expect(input.value).toBe(""); }); - it("defaults to agentic retrieval enabled and explains the toggle", async () => { + it("defaults to deep search enabled and explains the toggle", async () => { const user = userEvent.setup(); render(React.createElement(ChatComposer)); - const toggle = screen.getByRole("button", { - name: "Toggle agentic retrieval", + const toggle = screen.getByRole("checkbox", { + name: "Deep search", }); - expect(toggle.getAttribute("aria-pressed")).toBe("true"); + expect(toggle.getAttribute("aria-checked")).toBe("true"); await user.hover(toggle); const tooltip = await screen.findByRole("tooltip"); expect(tooltip.textContent).toContain( - "Agentic retrieval plans document selection and navigation", + "Deep search plans document selection and navigation", ); }); - it("sends useAgentic false after toggling agentic retrieval off", async () => { + it("sends useAgentic false after toggling deep search off", async () => { const user = userEvent.setup(); const onSend = vi.fn(); render(React.createElement(ChatComposer, { onSend })); - const toggle = screen.getByRole("button", { - name: "Toggle agentic retrieval", + const toggle = screen.getByRole("checkbox", { + name: "Deep search", }); await user.click(toggle); - expect(toggle.getAttribute("aria-pressed")).toBe("false"); + expect(toggle.getAttribute("aria-checked")).toBe("false"); const input = getComposerTextArea(); await user.type(input, "Quick summary"); diff --git a/src/components/chat-composer.tsx b/src/components/chat-composer.tsx index e9ee0859..4aab2633 100644 --- a/src/components/chat-composer.tsx +++ b/src/components/chat-composer.tsx @@ -10,9 +10,10 @@ import { type MouseEvent, type ReactElement, } from "react"; -import { BarChart3, FileText, Plus, Send, Sparkles } from "lucide-react"; +import { BarChart3, FileText, Plus, Send } from "lucide-react"; import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; import { DropdownMenu, DropdownMenuContent, @@ -192,28 +193,29 @@ export function ChatComposer({ - + + setUseAgentic(checked === true) + } + /> + Deep search + - Agentic retrieval plans document selection and navigation - for more thorough answers. Turn off for faster classic - search. + Deep search plans document selection and navigation for + more thorough answers. Turn off for faster classic search. From 76ede0fddeb206be693472659be490bf5dcc7732 Mon Sep 17 00:00:00 2001 From: cqboy1993 <167045138+EricNGOntos@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:55:55 +0800 Subject: [PATCH 03/11] feat(memory): extract and persist fluid memory after chat turns (#166) Add workspace-scoped fluid memory tables, lexical dedup token index, QStash extraction workflow, and fire-and-forget trigger after successful chat answers so durable user insights are stored for later cognition. Co-authored-by: Cursor --- .gitignore | 3 + drizzle/0013_first_magus.sql | 30 + drizzle/0014_messy_apocalypse.sql | 13 + drizzle/meta/0013_snapshot.json | 1185 +++++++++++++++ drizzle/meta/0014_snapshot.json | 1306 +++++++++++++++++ drizzle/meta/_journal.json | 14 + src/app/api/memory/extract/route.ts | 27 + src/domains/chat/route-answer.ts | 13 +- src/domains/chat/route-service.test.ts | 16 +- src/domains/memory/extract-trigger.ts | 43 + src/domains/memory/extract-workflow.ts | 169 +++ src/domains/memory/extraction-model.ts | 51 + src/domains/memory/prompts.test.ts | 35 + src/domains/memory/prompts.ts | 285 ++++ src/domains/memory/repository.ts | 254 ++++ src/domains/memory/resolve-operations.test.ts | 378 +++++ src/domains/memory/resolve-operations.ts | 314 ++++ src/domains/memory/search-index.test.ts | 94 ++ src/domains/memory/search-index.ts | 84 ++ src/domains/memory/service.ts | 54 + src/domains/memory/types.ts | 101 ++ src/infrastructure/db/schema.ts | 138 ++ 22 files changed, 4605 insertions(+), 2 deletions(-) create mode 100644 drizzle/0013_first_magus.sql create mode 100644 drizzle/0014_messy_apocalypse.sql create mode 100644 drizzle/meta/0013_snapshot.json create mode 100644 drizzle/meta/0014_snapshot.json create mode 100644 src/app/api/memory/extract/route.ts create mode 100644 src/domains/memory/extract-trigger.ts create mode 100644 src/domains/memory/extract-workflow.ts create mode 100644 src/domains/memory/extraction-model.ts create mode 100644 src/domains/memory/prompts.test.ts create mode 100644 src/domains/memory/prompts.ts create mode 100644 src/domains/memory/repository.ts create mode 100644 src/domains/memory/resolve-operations.test.ts create mode 100644 src/domains/memory/resolve-operations.ts create mode 100644 src/domains/memory/search-index.test.ts create mode 100644 src/domains/memory/search-index.ts create mode 100644 src/domains/memory/service.ts create mode 100644 src/domains/memory/types.ts diff --git a/.gitignore b/.gitignore index 2a2a9f19..f78db312 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,9 @@ .DS_Store *.pem .repos/effect +.tmp/ +.cursor/ +.agent/ # debug npm-debug.log* diff --git a/drizzle/0013_first_magus.sql b/drizzle/0013_first_magus.sql new file mode 100644 index 00000000..7aa2e5c5 --- /dev/null +++ b/drizzle/0013_first_magus.sql @@ -0,0 +1,30 @@ +CREATE TABLE "fluid_memory_items" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "kind" text NOT NULL, + "payload" jsonb NOT NULL, + "abstract_l0" text NOT NULL, + "overview_l1" text NOT NULL, + "source_message_id" uuid, + "confidence" double precision NOT NULL, + "status" text NOT NULL, + "version" integer DEFAULT 1 NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "memory_diffs" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "source_message_id" uuid, + "operations" jsonb NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "fluid_memory_items" ADD CONSTRAINT "fluid_memory_items_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "fluid_memory_items" ADD CONSTRAINT "fluid_memory_items_source_message_id_chat_messages_id_fk" FOREIGN KEY ("source_message_id") REFERENCES "public"."chat_messages"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "memory_diffs" ADD CONSTRAINT "memory_diffs_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "memory_diffs" ADD CONSTRAINT "memory_diffs_source_message_id_chat_messages_id_fk" FOREIGN KEY ("source_message_id") REFERENCES "public"."chat_messages"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "fluid_memory_items_workspace_status_idx" ON "fluid_memory_items" USING btree ("workspace_id","status");--> statement-breakpoint +CREATE INDEX "fluid_memory_items_workspace_kind_idx" ON "fluid_memory_items" USING btree ("workspace_id","kind");--> statement-breakpoint +CREATE INDEX "memory_diffs_workspace_created_idx" ON "memory_diffs" USING btree ("workspace_id","created_at"); \ No newline at end of file diff --git a/drizzle/0014_messy_apocalypse.sql b/drizzle/0014_messy_apocalypse.sql new file mode 100644 index 00000000..9233999e --- /dev/null +++ b/drizzle/0014_messy_apocalypse.sql @@ -0,0 +1,13 @@ +CREATE TABLE "fluid_memory_tokens" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "item_id" uuid NOT NULL, + "kind" text NOT NULL, + "token" text NOT NULL, + "frequency" integer DEFAULT 1 NOT NULL +); +--> statement-breakpoint +ALTER TABLE "fluid_memory_tokens" ADD CONSTRAINT "fluid_memory_tokens_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "fluid_memory_tokens" ADD CONSTRAINT "fluid_memory_tokens_item_id_fluid_memory_items_id_fk" FOREIGN KEY ("item_id") REFERENCES "public"."fluid_memory_items"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "fluid_memory_tokens_lookup_idx" ON "fluid_memory_tokens" USING btree ("workspace_id","kind","token");--> statement-breakpoint +CREATE INDEX "fluid_memory_tokens_item_idx" ON "fluid_memory_tokens" USING btree ("item_id"); \ No newline at end of file diff --git a/drizzle/meta/0013_snapshot.json b/drizzle/meta/0013_snapshot.json new file mode 100644 index 00000000..f46aa789 --- /dev/null +++ b/drizzle/meta/0013_snapshot.json @@ -0,0 +1,1185 @@ +{ + "id": "22b44d7b-bd80-4378-9789-756b55c75b0f", + "prevId": "a2a2f9a9-d567-4413-89d8-fa714b994351", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_memory_items": { + "name": "fluid_memory_items", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "abstract_l0": { + "name": "abstract_l0", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "overview_l1": { + "name": "overview_l1", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "fluid_memory_items_workspace_status_idx": { + "name": "fluid_memory_items_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fluid_memory_items_workspace_kind_idx": { + "name": "fluid_memory_items_workspace_kind_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_memory_items_workspace_id_workspaces_id_fk": { + "name": "fluid_memory_items_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_memory_items_source_message_id_chat_messages_id_fk": { + "name": "fluid_memory_items_source_message_id_chat_messages_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_diffs": { + "name": "memory_diffs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memory_diffs_workspace_created_idx": { + "name": "memory_diffs_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_diffs_workspace_id_workspaces_id_fk": { + "name": "memory_diffs_workspace_id_workspaces_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memory_diffs_source_message_id_chat_messages_id_fk": { + "name": "memory_diffs_source_message_id_chat_messages_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parsed_document_sync_leases": { + "name": "parsed_document_sync_leases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_token": { + "name": "lease_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_reason": { + "name": "release_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parsed_document_sync_leases_token_idx": { + "name": "parsed_document_sync_leases_token_idx", + "columns": [ + { + "expression": "lease_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_active_idx": { + "name": "parsed_document_sync_leases_active_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_workspace_active_idx": { + "name": "parsed_document_sync_leases_workspace_active_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_document_active_idx": { + "name": "parsed_document_sync_leases_document_active_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parsed_document_sync_leases_workspace_id_workspaces_id_fk": { + "name": "parsed_document_sync_leases_workspace_id_workspaces_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parsed_document_sync_leases_source_id_sources_id_fk": { + "name": "parsed_document_sync_leases_source_id_sources_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/0014_snapshot.json b/drizzle/meta/0014_snapshot.json new file mode 100644 index 00000000..4fe0e5f5 --- /dev/null +++ b/drizzle/meta/0014_snapshot.json @@ -0,0 +1,1306 @@ +{ + "id": "72bfb4fe-4048-4dce-a5fa-ce8eea880aa3", + "prevId": "22b44d7b-bd80-4378-9789-756b55c75b0f", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_memory_items": { + "name": "fluid_memory_items", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "abstract_l0": { + "name": "abstract_l0", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "overview_l1": { + "name": "overview_l1", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "fluid_memory_items_workspace_status_idx": { + "name": "fluid_memory_items_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fluid_memory_items_workspace_kind_idx": { + "name": "fluid_memory_items_workspace_kind_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_memory_items_workspace_id_workspaces_id_fk": { + "name": "fluid_memory_items_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_memory_items_source_message_id_chat_messages_id_fk": { + "name": "fluid_memory_items_source_message_id_chat_messages_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_memory_tokens": { + "name": "fluid_memory_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "item_id": { + "name": "item_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "frequency": { + "name": "frequency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + } + }, + "indexes": { + "fluid_memory_tokens_lookup_idx": { + "name": "fluid_memory_tokens_lookup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fluid_memory_tokens_item_idx": { + "name": "fluid_memory_tokens_item_idx", + "columns": [ + { + "expression": "item_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_memory_tokens_workspace_id_workspaces_id_fk": { + "name": "fluid_memory_tokens_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_memory_tokens", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_memory_tokens_item_id_fluid_memory_items_id_fk": { + "name": "fluid_memory_tokens_item_id_fluid_memory_items_id_fk", + "tableFrom": "fluid_memory_tokens", + "tableTo": "fluid_memory_items", + "columnsFrom": [ + "item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_diffs": { + "name": "memory_diffs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memory_diffs_workspace_created_idx": { + "name": "memory_diffs_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_diffs_workspace_id_workspaces_id_fk": { + "name": "memory_diffs_workspace_id_workspaces_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memory_diffs_source_message_id_chat_messages_id_fk": { + "name": "memory_diffs_source_message_id_chat_messages_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parsed_document_sync_leases": { + "name": "parsed_document_sync_leases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_token": { + "name": "lease_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_reason": { + "name": "release_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parsed_document_sync_leases_token_idx": { + "name": "parsed_document_sync_leases_token_idx", + "columns": [ + { + "expression": "lease_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_active_idx": { + "name": "parsed_document_sync_leases_active_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_workspace_active_idx": { + "name": "parsed_document_sync_leases_workspace_active_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_document_active_idx": { + "name": "parsed_document_sync_leases_document_active_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parsed_document_sync_leases_workspace_id_workspaces_id_fk": { + "name": "parsed_document_sync_leases_workspace_id_workspaces_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parsed_document_sync_leases_source_id_sources_id_fk": { + "name": "parsed_document_sync_leases_source_id_sources_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 93f0cedb..b9be4f54 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -92,6 +92,20 @@ "when": 1783231408481, "tag": "0012_lazy_wendell_rand", "breakpoints": true + }, + { + "idx": 13, + "version": "7", + "when": 1788247035130, + "tag": "0013_first_magus", + "breakpoints": true + }, + { + "idx": 14, + "version": "7", + "when": 1788252035435, + "tag": "0014_messy_apocalypse", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/app/api/memory/extract/route.ts b/src/app/api/memory/extract/route.ts new file mode 100644 index 00000000..ee25dbe4 --- /dev/null +++ b/src/app/api/memory/extract/route.ts @@ -0,0 +1,27 @@ +import { serve } from "@upstash/workflow/nextjs" + +import { + normalizeMemoryExtractPayload, + runMemoryExtractWorkflow, + type MemoryExtractPayload, +} from "@/domains/memory/extract-workflow" +import { logger } from "@/lib/logger" + +export const { POST } = serve( + async (context) => { + const payload = normalizeMemoryExtractPayload(context.requestPayload) + if (!payload) { + logger.warn("memory: extract workflow received invalid payload") + return + } + await runMemoryExtractWorkflow({ context, payload }) + }, + { + failureFunction: async ({ context, failResponse }) => { + logger.error("memory: extract workflow failed", { + payload: context.requestPayload, + failResponse, + }) + }, + }, +) diff --git a/src/domains/chat/route-answer.ts b/src/domains/chat/route-answer.ts index 73cb38fd..cc579b05 100644 --- a/src/domains/chat/route-answer.ts +++ b/src/domains/chat/route-answer.ts @@ -20,6 +20,7 @@ import { type ChatTurnValue, } from "@/domains/chat/service" import { chatTurnPersistence } from "@/domains/chat/chat-turn-persistence" +import { triggerMemoryExtraction } from "@/domains/memory/extract-trigger" import { startBackgroundReconciliation } from "@/domains/sources/background-reconcile" import { BlobParsedDocumentStorage } from "@/domains/sources/parsed-document-blob-storage" import { sourceWorkflowRuntime } from "@/domains/sources/workflow-runtime" @@ -196,7 +197,17 @@ const answerChatEffect = (input: AnswerChatInput) => return Either.match(result, { onLeft: (error): RouteResponse => routeResult.error(error.status, error.message), - onRight: (value): RouteResponse => routeResult.ok(value), + onRight: (value): RouteResponse => { + // Fire-and-forget: extract fluid memory from this turn without + // blocking the chat response. + void triggerMemoryExtraction({ + workspaceId: workspace.id, + threadId: value.threadId, + userMessageId: value.messages[0].id, + assistantMessageId: value.messages[1].id, + }) + return routeResult.ok(value) + }, }) }) diff --git a/src/domains/chat/route-service.test.ts b/src/domains/chat/route-service.test.ts index e71d38f1..02eceb9b 100644 --- a/src/domains/chat/route-service.test.ts +++ b/src/domains/chat/route-service.test.ts @@ -25,6 +25,7 @@ const mocks = vi.hoisted(() => ({ parsedStorageWriteAsset: vi.fn(), softDeleteChatThread: vi.fn(), startBackgroundReconciliation: vi.fn(), + triggerMemoryExtraction: vi.fn(), })) vi.mock("ai", async (importOriginal) => { @@ -62,6 +63,10 @@ vi.mock("@/domains/sources/background-reconcile", () => ({ startBackgroundReconciliation: mocks.startBackgroundReconciliation, })) +vi.mock("@/domains/memory/extract-trigger", () => ({ + triggerMemoryExtraction: mocks.triggerMemoryExtraction, +})) + vi.mock("@/domains/sources/workflow-runtime", () => ({ sourceWorkflowRuntime: { listForWorkspace: mocks.listSourcesForWorkspace, @@ -836,7 +841,10 @@ describe("chat route services", () => { mocks.handleChatTurn.mockResolvedValue( Either.right({ threadId: "thread_1", - messages: [], + messages: [ + { id: "message_user", role: "user", content: "Summarize it" }, + { id: "message_assistant", role: "assistant", content: "Summary" }, + ], }), ) @@ -845,6 +853,12 @@ describe("chat route services", () => { }) expect(result.status).toBe(200) + expect(mocks.triggerMemoryExtraction).toHaveBeenCalledWith({ + workspaceId: workspace.id, + threadId: "thread_1", + userMessageId: "message_user", + assistantMessageId: "message_assistant", + }) expect(mocks.startBackgroundReconciliation).toHaveBeenCalledWith( workspace.id, parsingSource.id, diff --git a/src/domains/memory/extract-trigger.ts b/src/domains/memory/extract-trigger.ts new file mode 100644 index 00000000..35b64675 --- /dev/null +++ b/src/domains/memory/extract-trigger.ts @@ -0,0 +1,43 @@ +import "server-only" + +import { Client } from "@upstash/workflow" + +import type { MemoryExtractPayload } from "./extract-workflow" +import { logger } from "@/lib/logger" + +function resolveBaseURL(): string { + return process.env.NOTEBOOK_PUBLIC_URL ?? "http://localhost:3000" +} + +/** + * Fire-and-forget trigger for the post-turn fluid-memory extraction + * workflow. Each turn is uniquely keyed by its message ids, so unlike the + * source reconcile trigger no cooldown/dedup guard is needed; QStash + * retries cover transient delivery failures. + */ +export async function triggerMemoryExtraction( + payload: MemoryExtractPayload, +): Promise { + const token = process.env.QSTASH_TOKEN + if (!token) { + logger.warn("memory: skipping extraction — QSTASH_TOKEN not set", { + workspaceId: payload.workspaceId, + assistantMessageId: payload.assistantMessageId, + }) + return + } + + try { + await new Client({ token }).trigger({ + url: `${resolveBaseURL()}/api/memory/extract`, + body: payload, + retries: 3, + }) + } catch (error) { + logger.error("memory: failed to trigger extraction workflow", { + workspaceId: payload.workspaceId, + assistantMessageId: payload.assistantMessageId, + message: error instanceof Error ? error.message : String(error), + }) + } +} diff --git a/src/domains/memory/extract-workflow.ts b/src/domains/memory/extract-workflow.ts new file mode 100644 index 00000000..aa6b5ee3 --- /dev/null +++ b/src/domains/memory/extract-workflow.ts @@ -0,0 +1,169 @@ +import "server-only" + +import type { WorkflowContext } from "@upstash/workflow" + +import { extractMemoryOperations } from "./extraction-model" +import { + summarizePayloadForContext, + type ExistingMemoryContextItem, +} from "./prompts" +import { resolveMemoryOperations } from "./resolve-operations" +import { tokenizeMemoryText } from "./search-index" +import { memoryService } from "./service" +import { fluidMemoryKinds, isFluidMemoryKind } from "./types" +import { chatThreadService } from "@/domains/chat/thread-service" +import { logger } from "@/lib/logger" + +export type MemoryExtractPayload = { + readonly workspaceId: string + readonly threadId: string + readonly userMessageId: string + readonly assistantMessageId: string +} + +type MemoryExtractWorkflowContext = Pick< + WorkflowContext, + "run" +> + +/** Prompt-context item that also carries status/payload for resolution. */ +type MemoryWorkflowItem = ExistingMemoryContextItem & { + readonly status: string + readonly payload: unknown +} + +/** Per-kind cap on lexical neighbors fed into the merge-decision prompt. */ +const DEDUP_CANDIDATES_PER_KIND = 8 + +export function normalizeMemoryExtractPayload( + raw: unknown, +): MemoryExtractPayload | null { + if (!raw || typeof raw !== "object") return null + const record = raw as Record + const workspaceId = getNonEmptyString(record.workspaceId) + const threadId = getNonEmptyString(record.threadId) + const userMessageId = getNonEmptyString(record.userMessageId) + const assistantMessageId = getNonEmptyString(record.assistantMessageId) + if (!workspaceId || !threadId || !userMessageId || !assistantMessageId) { + return null + } + return { workspaceId, threadId, userMessageId, assistantMessageId } +} + +export async function runMemoryExtractWorkflow(input: { + readonly context: MemoryExtractWorkflowContext + readonly payload: MemoryExtractPayload +}): Promise { + const { context, payload } = input + + const turn = await context.run("load-turn", async () => { + const messages = await chatThreadService.listMessages( + payload.workspaceId, + payload.threadId, + ) + const userMessage = messages?.find( + (message) => message.id === payload.userMessageId, + ) + const assistantMessage = messages?.find( + (message) => message.id === payload.assistantMessageId, + ) + if (!userMessage || !assistantMessage) return null + return { + userText: userMessage.content, + assistantText: assistantMessage.content, + referencedDocumentIds: collectCitationDocumentIds( + assistantMessage.citations, + ), + } + }) + if (!turn) { + logger.warn("memory: extract skipped — turn messages not found", { + workspaceId: payload.workspaceId, + threadId: payload.threadId, + }) + return + } + + const existingItems = await context.run("retrieve-candidates", async () => { + const queryTokens = tokenizeMemoryText(turn.userText).map( + (entry) => entry.token, + ) + if (queryTokens.length === 0) return [] + + const byId = new Map() + for (const kind of fluidMemoryKinds) { + const items = await memoryService.findDedupCandidates( + payload.workspaceId, + kind, + queryTokens, + DEDUP_CANDIDATES_PER_KIND, + ) + for (const item of items) { + if (!isFluidMemoryKind(item.kind) || byId.has(item.id)) continue + byId.set(item.id, { + id: item.id, + kind: item.kind, + status: item.status, + payload: item.payload, + abstractL0: item.abstractL0, + payloadSummary: summarizePayloadForContext(item.kind, item.payload), + }) + } + } + return [...byId.values()] + }) + + const operations = await context.run("extract-operations", () => + extractMemoryOperations({ + workspaceId: payload.workspaceId, + userText: turn.userText, + assistantText: turn.assistantText, + referencedDocumentIds: turn.referencedDocumentIds, + existingItems, + }), + ) + if (!operations) return + + const applied = await context.run("apply-operations", async () => { + const resolved = resolveMemoryOperations({ + operations, + existingItems, + referencedDocumentIds: turn.referencedDocumentIds, + }) + if (resolved.length === 0) return null + return memoryService.applyOperations( + payload.workspaceId, + payload.assistantMessageId, + resolved, + ) + }) + + logger.info("memory: extract workflow finished", { + workspaceId: payload.workspaceId, + threadId: payload.threadId, + assistantMessageId: payload.assistantMessageId, + candidateCount: existingItems.length, + appliedOperations: applied?.map((operation) => operation.op) ?? [], + }) +} + +function collectCitationDocumentIds(citations: unknown): string[] { + if (!Array.isArray(citations)) return [] + const ids = new Set() + for (const citation of citations) { + if (!citation || typeof citation !== "object") continue + const source = (citation as Record).source + if (!source || typeof source !== "object") continue + const documentId = (source as Record).documentId + if (typeof documentId === "string" && documentId.length > 0) { + ids.add(documentId) + } + } + return [...ids] +} + +function getNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 + ? value + : null +} diff --git a/src/domains/memory/extraction-model.ts b/src/domains/memory/extraction-model.ts new file mode 100644 index 00000000..fa490115 --- /dev/null +++ b/src/domains/memory/extraction-model.ts @@ -0,0 +1,51 @@ +import "server-only" + +import { generateObject } from "ai" + +import { + buildMemoryExtractionPrompt, + memoryOperationsSchema, + type ExistingMemoryContextItem, + type MemoryOperations, +} from "./prompts" +import { CHAT_MODEL } from "@/lib/ai" +import { summarizeUnknownError } from "@/lib/format-log-value" +import { logger } from "@/lib/logger" + +const MEMORY_EXTRACTION_MODEL = process.env.MEMORY_EXTRACTION_MODEL ?? CHAT_MODEL + +/** + * One structured-output call: turn + existing active memories in, typed + * operations out. Best-effort by design — this runs as a background job, + * so a model failure skips the turn (logged) instead of degrading through + * fallbacks; the insight typically resurfaces in a later turn. + */ +export async function extractMemoryOperations(input: { + readonly workspaceId: string + readonly userText: string + readonly assistantText: string + readonly referencedDocumentIds: readonly string[] + readonly existingItems: readonly ExistingMemoryContextItem[] +}): Promise { + try { + const response = await generateObject({ + model: MEMORY_EXTRACTION_MODEL, + schema: memoryOperationsSchema, + messages: [ + { + role: "user", + content: buildMemoryExtractionPrompt(input), + }, + ], + }) + return response.object + } catch (error) { + logger.warn("memory: extraction model call failed; skipping turn", { + workspaceId: input.workspaceId, + model: MEMORY_EXTRACTION_MODEL, + existingItemCount: input.existingItems.length, + error: summarizeUnknownError(error), + }) + return null + } +} diff --git a/src/domains/memory/prompts.test.ts b/src/domains/memory/prompts.test.ts new file mode 100644 index 00000000..2ae15ff2 --- /dev/null +++ b/src/domains/memory/prompts.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest" + +import { buildMemoryExtractionPrompt } from "./prompts" + +describe("buildMemoryExtractionPrompt", () => { + const prompt = buildMemoryExtractionPrompt({ + userText: "毛利率是核心。", + assistantText: "明白。", + referencedDocumentIds: ["doc-1"], + existingItems: [], + }) + + it("keeps main instructions domain-agnostic", () => { + const main = prompt.slice(0, prompt.indexOf("## Illustrative examples")) + expect(main).not.toMatch(/gross margin|市盈率|\bPE\b|Fed|美联储|NVIDIA|英伟达/i) + expect(main).toContain("Write every free-text value") + expect(main).toMatch(/same language the\s+USER wrote/) + }) + + it("keeps illustrative examples in a separate section", () => { + expect(prompt).toContain("## Illustrative examples (finance vertical") + expect(prompt).toContain("not exhaustive, not required vocabulary") + const examples = prompt.slice( + prompt.indexOf("## Illustrative examples"), + prompt.indexOf("## Output JSON schema"), + ) + expect(examples).toContain("finance vertical") + expect(examples).toContain("do not force the conversation into this domain") + }) + + it("still injects turn context after the fixed blocks", () => { + expect(prompt).toContain("[user]\n毛利率是核心。") + expect(prompt).toContain("doc-1") + }) +}) diff --git a/src/domains/memory/prompts.ts b/src/domains/memory/prompts.ts new file mode 100644 index 00000000..e2fbeb0c --- /dev/null +++ b/src/domains/memory/prompts.ts @@ -0,0 +1,285 @@ +import { z } from "zod" + +import { + decisionRulePayloadSchema, + entityOfInterestPayloadSchema, + indicatorPreferencePayloadSchema, + stancePayloadSchema, + type FluidMemoryKind, +} from "./types" + +/** + * LLM contract for fluid-memory extraction. + * + * Design borrowed from OpenViking's session-commit extraction (schema-driven + * typed operations, prefetch-then-decide), reimplemented as a single + * structured-output call: the model sees the turn plus lexically retrieved + * dedup candidates and directly outputs per-kind operations + * (create / skip / merge / deprecate), mirroring OpenViking's generated + * operations model without its ReAct tool loop. + */ + +const decisionSchema = z.object({ + op: z.enum(["create", "skip", "merge", "deprecate"]), + targetItemId: z.preprocess( + (value) => (value === null ? undefined : value), + z + .string() + .optional() + .describe( + "Required for merge/deprecate: the id of the existing memory item this operation targets. Omit for create/skip.", + ), + ), + reason: z.preprocess( + (value) => (value === null ? undefined : value), + z + .string() + .optional() + .describe("Short justification, especially for skip/merge/deprecate."), + ), +}) + +const memorySidecarFields = { + abstractL0: z + .string() + .min(1) + .describe("One line, <= 30 words: the essence of this insight."), + overviewL1: z + .string() + .min(1) + .describe("2-3 sentences: what it means and when it applies."), + confidence: z + .number() + .min(0) + .max(1) + .describe("How explicitly the user stated this (1 = explicit)."), + decision: decisionSchema, +} + +const stanceEntrySchema = z.preprocess((value) => { + if (!value || typeof value !== "object" || Array.isArray(value)) return value + const record = value as Record + // Models sometimes emit "name" for a stance; the contract field is "statement". + if ( + (typeof record.statement !== "string" || record.statement.length === 0) && + typeof record.name === "string" && + record.name.length > 0 + ) { + const { name, ...rest } = record + return { ...rest, statement: name } + } + return value +}, stancePayloadSchema.extend(memorySidecarFields)) + +const entityEntrySchema = z.preprocess((value) => { + if (!value || typeof value !== "object" || Array.isArray(value)) return value + const record = value as Record + // Keep provenance reason required; if the model omitted it, fall back to L0. + if ( + (typeof record.reason !== "string" || record.reason.length === 0) && + typeof record.abstractL0 === "string" && + record.abstractL0.length > 0 + ) { + return { ...record, reason: record.abstractL0 } + } + return value +}, entityOfInterestPayloadSchema.extend(memorySidecarFields)) + +export const memoryOperationsSchema = z.object({ + indicatorPrefs: z + .array(indicatorPreferencePayloadSchema.extend(memorySidecarFields)) + .default([]), + stances: z.array(stanceEntrySchema).default([]), + decisionRules: z + .array(decisionRulePayloadSchema.extend(memorySidecarFields)) + .default([]), + entities: z.array(entityEntrySchema).default([]), +}) + +export type MemoryOperations = z.infer + +export type ExistingMemoryContextItem = { + readonly id: string + readonly kind: FluidMemoryKind + readonly abstractL0: string + readonly payloadSummary: string +} + +/** Structural output shape only — no domain content. */ +const OUTPUT_SCHEMA_BLOCK = `{ + "indicatorPrefs": [{ + "name": "string", + "aliases": ["string"], + "definition": "string", + "polarity": "higher_better|lower_better|context", + "importance": "core|secondary", + "formulaHint": "string (optional — omit if none)", + "abstractL0": "string", + "overviewL1": "string", + "confidence": 0.0, + "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } + }], + "stances": [{ + "statement": "string (the stance text; do not use a name field)", + "scope": "string", + "rationale": "string", + "abstractL0": "string", + "overviewL1": "string", + "confidence": 0.0, + "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } + }], + "decisionRules": [{ + "when": "string", + "then": "string", + "priority": "high|medium|low", + "rationale": "string", + "abstractL0": "string", + "overviewL1": "string", + "confidence": 0.0, + "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } + }], + "entities": [{ + "name": "string", + "ticker": "string optional", + "aliases": ["string"], + "knowhereDocumentIds": ["only ids listed under REFERENCED DOCUMENT IDS"], + "reason": "string", + "abstractL0": "string", + "overviewL1": "string", + "confidence": 0.0, + "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } + }] +}` + +/** + * Illustrative only — kept separate from the main instructions so the model + * does not treat these domain phrases as required vocabulary. + * Finance is the first vertical; add other industry blocks here later if needed. + */ +const ILLUSTRATIVE_EXAMPLES_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) + +These show shape and judgement only. Extract whatever the user actually said; +do not force the conversation into this domain or these metric names. + +- indicatorPref: user says a named metric they repeatedly use to judge quality + (shape: name + short definition + polarity + importance). Same idea applies + outside finance (any recurring evaluation metric). +- stance: user states a durable judgement frame that changes how evidence is + weighted (e.g. long-horizon vs short-horizon). +- decisionRule: user states a reusable when → then discipline over their metrics. +- entity: user says they actively track a named company/issuer and why. +- skip: a one-off factual question about a page/number in a document, small talk, + or an assistant suggestion the user did not endorse.` + +/** Domain-agnostic extraction instructions. */ +const MAIN_INSTRUCTIONS_BLOCK = `You maintain a user's FLUID MEMORY: durable insights about how this user thinks, extracted from their conversation with an AI analyst. + +Document facts live elsewhere (crystal memory). Never extract document facts, retrieved numbers, or page content as fluid memory. + +## What to extract + +Extract ONLY these four kinds, and ONLY when the turn gives real evidence from the USER: + +- indicatorPrefs — recurring metrics or criteria the user uses to evaluate things. + Fields: name, aliases, definition, polarity (higher_better | lower_better | context), + importance (core | secondary), optional formulaHint. +- stances — durable positions that shape how the user weighs evidence. + Fields: statement (required; do not invent a "name" field), scope, rationale. +- decisionRules — reusable when → then disciplines the user stated or clearly endorsed. + Fields: when, then, priority (high | medium | low), rationale. +- entities — named subjects the user is actively tracking. + Fields: name, optional ticker, aliases, reason (required), knowhereDocumentIds + (only from REFERENCED DOCUMENT IDS below; never invent ids). + +## Language + +- Keep this instruction set and enum/field names in English. +- Write every free-text value (name, definition, statement, when/then, reason, + abstractL0, overviewL1, aliases the user used, etc.) in the same language the + USER wrote in this turn. Do not translate the user's terms into English unless + the user themselves used English. + +## Decision rules + +- Extract only durable, reusable insights about the USER. +- Skip one-off questions, document facts, small talk, and assistant claims the user did not endorse. +- For every candidate, choose exactly one op against EXISTING MEMORIES: + - create — genuinely new + - skip — already covered, or too weak/ephemeral + - merge — same insight refined; emit the full merged fields and set targetItemId + - deprecate — user explicitly reversed a stored item; set targetItemId +- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. +- Prefer one record per insight. If a preference already encodes how a metric should be read, do not also invent a near-duplicate decisionRule unless the user stated an explicit when → then action. +- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when the user stated it explicitly. +- Omit optional fields instead of setting them to null. +- If nothing is worth remembering, return all four arrays empty.` + +export function buildMemoryExtractionPrompt(input: { + readonly userText: string + readonly assistantText: string + readonly referencedDocumentIds: readonly string[] + readonly existingItems: readonly ExistingMemoryContextItem[] +}): string { + const existingBlock = + input.existingItems.length === 0 + ? "(no existing memories yet)" + : input.existingItems + .map( + (item) => + `- [${item.kind}] id=${item.id} :: ${item.abstractL0} :: ${item.payloadSummary}`, + ) + .join("\n") + + const documentsBlock = + input.referencedDocumentIds.length === 0 + ? "(no documents referenced in this turn)" + : input.referencedDocumentIds.join(", ") + + return `${MAIN_INSTRUCTIONS_BLOCK} + +${ILLUSTRATIVE_EXAMPLES_BLOCK} + +## Output JSON schema (follow exactly; do not invent fields) + +${OUTPUT_SCHEMA_BLOCK} + +## EXISTING MEMORIES + +${existingBlock} + +## REFERENCED DOCUMENT IDS + +${documentsBlock} + +## CONVERSATION TURN + +[user] +${input.userText} + +[assistant] +${input.assistantText}` +} + +export function summarizePayloadForContext( + kind: FluidMemoryKind, + payload: unknown, +): string { + if (!payload || typeof payload !== "object") return "" + const record = payload as Record + switch (kind) { + case "indicator_pref": + return [record.name, record.definition] + .filter((part) => typeof part === "string" && part.length > 0) + .join(" — ") + case "stance": + return typeof record.statement === "string" ? record.statement : "" + case "decision_rule": + return [record.when, record.then] + .filter((part) => typeof part === "string" && part.length > 0) + .join(" => ") + case "entity_of_interest": + return [record.name, record.ticker] + .filter((part) => typeof part === "string" && part.length > 0) + .join(" ") + } +} diff --git a/src/domains/memory/repository.ts b/src/domains/memory/repository.ts new file mode 100644 index 00000000..f7f6b794 --- /dev/null +++ b/src/domains/memory/repository.ts @@ -0,0 +1,254 @@ +import "server-only" + +import { and, eq, inArray, sql } from "drizzle-orm" +import { Effect } from "effect" + +import { toDiffOperation, type ResolvedMemoryOperation } from "./resolve-operations" +import { buildMemoryItemTokens } from "./search-index" +import type { + FluidMemoryKind, + FluidMemoryPayload, + MemoryDiffOperation, +} from "./types" +import { DbClient } from "@/infrastructure/db" +import { + fluidMemoryItems, + fluidMemoryTokens, + memoryDiffs, + type FluidMemoryItem, + type NewFluidMemoryToken, +} from "@/infrastructure/db/schema" + +type MemoryRepository = { + readonly findDedupCandidatesEffect: ( + workspaceId: string, + kind: FluidMemoryKind, + tokens: readonly string[], + limit: number, + ) => Effect.Effect + readonly applyOperationsEffect: ( + workspaceId: string, + sourceMessageId: string | null, + operations: readonly ResolvedMemoryOperation[], + ) => Effect.Effect +} + +type RawRowsResult = readonly Row[] | { readonly rows: readonly Row[] } + +/** + * Retrieve the most lexically-similar active items of one kind, ranked by + * idf-weighted token overlap computed entirely in SQL. Common tokens (high + * document frequency within this workspace + kind) are down-weighted so a + * shared rare term outranks several shared filler characters. + * + * Token rows only exist for active items (see schema invariant), so no + * status filter is needed here. + */ +const findDedupCandidatesEffect: MemoryRepository["findDedupCandidatesEffect"] = + (workspaceId, kind, tokens, limit) => + Effect.gen(function* () { + const db = yield* DbClient + if (tokens.length === 0 || limit <= 0) return [] + + const tokenList = sql.join( + tokens.map((token) => sql`${token}`), + sql`, `, + ) + + const scored = yield* Effect.promise(() => + db.execute<{ itemId: string }>(sql` + SELECT t.item_id AS "itemId", SUM(t.frequency::float8 / df.df) AS score + FROM fluid_memory_tokens t + JOIN ( + SELECT token, COUNT(DISTINCT item_id)::float8 AS df + FROM fluid_memory_tokens + WHERE workspace_id = ${workspaceId}::uuid + AND kind = ${kind} + AND token IN (${tokenList}) + GROUP BY token + ) df ON df.token = t.token + WHERE t.workspace_id = ${workspaceId}::uuid + AND t.kind = ${kind} + AND t.token IN (${tokenList}) + GROUP BY t.item_id + ORDER BY score DESC + LIMIT ${limit} + `), + ) + + const orderedIds = getRawRows(scored).map((row) => row.itemId) + if (orderedIds.length === 0) return [] + + const items = yield* Effect.promise(() => + db + .select() + .from(fluidMemoryItems) + .where(inArray(fluidMemoryItems.id, orderedIds)), + ) + const byId = new Map(items.map((item) => [item.id, item] as const)) + return orderedIds.flatMap((id) => { + const item = byId.get(id) + return item ? [item] : [] + }) + }) + +const applyOperationsEffect: MemoryRepository["applyOperationsEffect"] = ( + workspaceId, + sourceMessageId, + operations, +) => + Effect.gen(function* () { + const db = yield* DbClient + return yield* Effect.promise(() => + db.transaction(async (tx) => { + const diffOperations: MemoryDiffOperation[] = [] + + for (const operation of operations) { + switch (operation.op) { + case "create": { + const [inserted] = await tx + .insert(fluidMemoryItems) + .values({ + workspaceId, + kind: operation.kind, + payload: operation.payload, + abstractL0: operation.abstractL0, + overviewL1: operation.overviewL1, + sourceMessageId, + confidence: operation.confidence, + status: "active", + }) + .returning() + if (inserted?.id) { + const tokenRows = tokenRowsFor( + workspaceId, + inserted.id, + operation.kind, + operation.payload, + ) + if (tokenRows.length > 0) { + await tx.insert(fluidMemoryTokens).values(tokenRows) + } + } + diffOperations.push(toDiffOperation(operation, inserted?.id)) + break + } + case "merge": { + const [updated] = await tx + .update(fluidMemoryItems) + .set({ + payload: operation.payload, + abstractL0: operation.abstractL0, + overviewL1: operation.overviewL1, + confidence: operation.confidence, + sourceMessageId, + version: sql`${fluidMemoryItems.version} + 1`, + updatedAt: sql`now()`, + }) + .where( + and( + eq(fluidMemoryItems.id, operation.targetItemId), + eq(fluidMemoryItems.status, "active"), + ), + ) + .returning({ id: fluidMemoryItems.id }) + if (updated) { + await tx + .delete(fluidMemoryTokens) + .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) + const tokenRows = tokenRowsFor( + workspaceId, + operation.targetItemId, + operation.kind, + operation.payload, + ) + if (tokenRows.length > 0) { + await tx.insert(fluidMemoryTokens).values(tokenRows) + } + } + diffOperations.push( + updated + ? toDiffOperation(operation) + : { + op: "skip", + kind: operation.kind, + summary: operation.summary, + reason: "merge target no longer active", + }, + ) + break + } + case "deprecate": { + const [updated] = await tx + .update(fluidMemoryItems) + .set({ + status: "deprecated", + updatedAt: sql`now()`, + }) + .where( + and( + eq(fluidMemoryItems.id, operation.targetItemId), + eq(fluidMemoryItems.status, "active"), + ), + ) + .returning({ id: fluidMemoryItems.id }) + if (updated) { + await tx + .delete(fluidMemoryTokens) + .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) + } + diffOperations.push( + updated + ? toDiffOperation(operation) + : { + op: "skip", + kind: operation.kind, + summary: operation.summary, + reason: "deprecate target no longer active", + }, + ) + break + } + case "skip": + diffOperations.push(toDiffOperation(operation)) + break + } + } + + if (diffOperations.length > 0) { + await tx.insert(memoryDiffs).values({ + workspaceId, + sourceMessageId, + operations: [...diffOperations], + }) + } + + return diffOperations + }), + ) + }) + +export const memoryRepository: MemoryRepository = { + findDedupCandidatesEffect, + applyOperationsEffect, +} + +function tokenRowsFor( + workspaceId: string, + itemId: string, + kind: FluidMemoryKind, + payload: FluidMemoryPayload, +): NewFluidMemoryToken[] { + return buildMemoryItemTokens(kind, payload).map((token) => ({ + workspaceId, + itemId, + kind, + token: token.token, + frequency: token.frequency, + })) +} + +function getRawRows(value: RawRowsResult): readonly Row[] { + if (Array.isArray(value)) return value + return (value as { readonly rows: readonly Row[] }).rows +} diff --git a/src/domains/memory/resolve-operations.test.ts b/src/domains/memory/resolve-operations.test.ts new file mode 100644 index 00000000..9764ccfd --- /dev/null +++ b/src/domains/memory/resolve-operations.test.ts @@ -0,0 +1,378 @@ +import { describe, expect, it } from "vitest" + +import type { MemoryOperations } from "./prompts" +import { + resolveMemoryOperations, + toDiffOperation, +} from "./resolve-operations" + +const existingItems = [ + { + id: "item-1", + kind: "indicator_pref", + status: "active", + payload: { + name: "毛利率", + aliases: ["gross margin"], + definition: "毛利占营收的比例", + polarity: "higher_better", + importance: "core", + }, + }, + { id: "item-2", kind: "stance", status: "active" }, + { id: "item-3", kind: "stance", status: "deprecated" }, + { + id: "item-4", + kind: "entity_of_interest", + status: "active", + payload: { + name: "英伟达", + ticker: "NVDA", + aliases: ["NVIDIA"], + knowhereDocumentIds: ["doc-earlier"], + reason: "用户持续跟踪", + }, + }, +] as const + +function makeOperations( + overrides: Partial = {}, +): MemoryOperations { + return { + indicatorPrefs: [], + stances: [], + decisionRules: [], + entities: [], + ...overrides, + } +} + +function makeIndicatorEntry(decision: { + op: "create" | "skip" | "merge" | "deprecate" + targetItemId?: string + reason?: string +}) { + return { + name: "毛利率", + aliases: ["gross margin"], + definition: "毛利占营收的比例", + polarity: "higher_better" as const, + importance: "core" as const, + abstractL0: "用户看重毛利率", + overviewL1: "用户在分析公司时首先看毛利率。", + confidence: 0.9, + decision, + } +} + +describe("resolveMemoryOperations", () => { + it("passes create through and ignores a stray targetItemId", () => { + const resolved = resolveMemoryOperations({ + operations: makeOperations({ + indicatorPrefs: [ + makeIndicatorEntry({ op: "create", targetItemId: "item-1" }), + ], + }), + existingItems, + referencedDocumentIds: [], + }) + + expect(resolved).toHaveLength(1) + expect(resolved[0]).toMatchObject({ + op: "create", + kind: "indicator_pref", + payload: { + name: "毛利率", + aliases: ["gross margin"], + polarity: "higher_better", + }, + }) + expect(resolved[0]).not.toHaveProperty("targetItemId") + }) + + it("merges into an existing active item of the same kind", () => { + const resolved = resolveMemoryOperations({ + operations: makeOperations({ + indicatorPrefs: [ + makeIndicatorEntry({ op: "merge", targetItemId: "item-1" }), + ], + }), + existingItems, + referencedDocumentIds: [], + }) + + expect(resolved[0]).toMatchObject({ + op: "merge", + kind: "indicator_pref", + targetItemId: "item-1", + }) + }) + + it("downgrades merge to skip when the target is missing", () => { + const resolved = resolveMemoryOperations({ + operations: makeOperations({ + indicatorPrefs: [ + makeIndicatorEntry({ op: "merge", targetItemId: "item-999" }), + ], + }), + existingItems, + referencedDocumentIds: [], + }) + + expect(resolved[0]).toMatchObject({ + op: "skip", + kind: "indicator_pref", + }) + expect(resolved[0]).not.toHaveProperty("targetItemId") + }) + + it("downgrades merge to skip on kind mismatch", () => { + const resolved = resolveMemoryOperations({ + operations: makeOperations({ + indicatorPrefs: [ + makeIndicatorEntry({ op: "merge", targetItemId: "item-2" }), + ], + }), + existingItems, + referencedDocumentIds: [], + }) + + expect(resolved[0]?.op).toBe("skip") + }) + + it("downgrades merge to skip when the target is already deprecated", () => { + const resolved = resolveMemoryOperations({ + operations: makeOperations({ + stances: [ + { + statement: "长期持有，忽略短期波动", + scope: "投资", + rationale: "用户做长期投资", + abstractL0: "长期投资立场", + overviewL1: "用户强调长期持有。", + confidence: 1, + decision: { op: "merge", targetItemId: "item-3" }, + }, + ], + }), + existingItems, + referencedDocumentIds: [], + }) + + expect(resolved[0]?.op).toBe("skip") + }) + + it("keeps deprecate for an active target", () => { + const resolved = resolveMemoryOperations({ + operations: makeOperations({ + stances: [ + { + statement: "美联储短期观点不重要", + scope: "宏观", + rationale: "长期投资", + abstractL0: "不看美联储短期观点", + overviewL1: "用户认为美联储短期观点权重低。", + confidence: 1, + decision: { + op: "deprecate", + targetItemId: "item-2", + reason: "用户改口", + }, + }, + ], + }), + existingItems, + referencedDocumentIds: [], + }) + + expect(resolved[0]).toMatchObject({ + op: "deprecate", + targetItemId: "item-2", + reason: "用户改口", + }) + }) + + it("filters entity document ids to the turn's referenced documents", () => { + const resolved = resolveMemoryOperations({ + operations: makeOperations({ + entities: [ + { + name: "英伟达", + ticker: "NVDA", + aliases: ["NVIDIA"], + knowhereDocumentIds: ["doc-real", "doc-hallucinated"], + reason: "用户持续跟踪", + abstractL0: "用户关注英伟达", + overviewL1: "用户多次询问英伟达财报。", + confidence: 0.8, + decision: { op: "create" }, + }, + ], + }), + existingItems, + referencedDocumentIds: ["doc-real"], + }) + + expect(resolved[0]).toMatchObject({ + op: "create", + kind: "entity_of_interest", + payload: { + name: "英伟达", + knowhereDocumentIds: ["doc-real"], + }, + }) + }) + + it("unions stored document ids when merging an entity", () => { + const resolved = resolveMemoryOperations({ + operations: makeOperations({ + entities: [ + { + name: "英伟达", + ticker: "NVDA", + aliases: ["NVDA Corp"], + knowhereDocumentIds: ["doc-this-turn"], + reason: "用户持续跟踪", + abstractL0: "用户关注英伟达", + overviewL1: "用户多次询问英伟达财报。", + confidence: 0.9, + decision: { op: "merge", targetItemId: "item-4" }, + }, + ], + }), + existingItems, + referencedDocumentIds: ["doc-this-turn"], + }) + + expect(resolved[0]).toMatchObject({ + op: "merge", + targetItemId: "item-4", + payload: { + aliases: ["NVIDIA", "NVDA Corp"], + knowhereDocumentIds: ["doc-earlier", "doc-this-turn"], + }, + }) + }) + + it("unions aliases when merging an indicator preference", () => { + const resolved = resolveMemoryOperations({ + operations: makeOperations({ + indicatorPrefs: [ + { + name: "毛利率", + aliases: ["同比毛利率"], + definition: "毛利占营收的比例，也看同比", + polarity: "higher_better" as const, + importance: "core" as const, + abstractL0: "毛利率也看同比", + overviewL1: "用户补充了同比视角。", + confidence: 1, + decision: { op: "merge", targetItemId: "item-1" }, + }, + ], + }), + existingItems, + referencedDocumentIds: [], + }) + + expect(resolved[0]).toMatchObject({ + op: "merge", + targetItemId: "item-1", + payload: { + aliases: ["gross margin", "同比毛利率"], + }, + }) + }) + + it("skips create when the payload has no searchable tokens", () => { + const resolved = resolveMemoryOperations({ + operations: makeOperations({ + indicatorPrefs: [ + { + name: "!!!", + aliases: [], + definition: "???", + polarity: "context" as const, + importance: "secondary" as const, + abstractL0: "无效符号", + overviewL1: "无法检索。", + confidence: 0.1, + decision: { op: "create" }, + }, + ], + }), + existingItems, + referencedDocumentIds: [], + }) + + expect(resolved[0]).toMatchObject({ + op: "skip", + kind: "indicator_pref", + reason: "payload has no searchable tokens", + }) + }) + + it("skips merge when the merged payload has no searchable tokens", () => { + const resolved = resolveMemoryOperations({ + operations: makeOperations({ + stances: [ + { + statement: "!!!", + scope: "???", + rationale: "...", + abstractL0: "无效符号", + overviewL1: "无法检索。", + confidence: 0.1, + decision: { op: "merge", targetItemId: "item-2" }, + }, + ], + }), + existingItems, + referencedDocumentIds: [], + }) + + expect(resolved[0]).toMatchObject({ + op: "skip", + kind: "stance", + reason: "merged payload has no searchable tokens", + }) + }) +}) + +describe("toDiffOperation", () => { + it("records create with the inserted item id", () => { + const resolved = resolveMemoryOperations({ + operations: makeOperations({ + indicatorPrefs: [makeIndicatorEntry({ op: "create" })], + }), + existingItems, + referencedDocumentIds: [], + }) + + expect(toDiffOperation(resolved[0]!, "new-id")).toEqual({ + op: "create", + kind: "indicator_pref", + summary: "用户看重毛利率", + itemId: "new-id", + }) + }) + + it("records merge/deprecate with their target item id", () => { + const resolved = resolveMemoryOperations({ + operations: makeOperations({ + indicatorPrefs: [ + makeIndicatorEntry({ op: "merge", targetItemId: "item-1" }), + ], + }), + existingItems, + referencedDocumentIds: [], + }) + + expect(toDiffOperation(resolved[0]!)).toEqual({ + op: "merge", + kind: "indicator_pref", + summary: "用户看重毛利率", + itemId: "item-1", + }) + }) +}) diff --git a/src/domains/memory/resolve-operations.ts b/src/domains/memory/resolve-operations.ts new file mode 100644 index 00000000..ee1e6a42 --- /dev/null +++ b/src/domains/memory/resolve-operations.ts @@ -0,0 +1,314 @@ +import { buildMemoryItemTokens } from "./search-index" +import type { MemoryOperations } from "./prompts" +import { + parseFluidMemoryPayload, + type FluidMemoryKind, + type FluidMemoryPayload, + type MemoryDiffOperation, +} from "./types" + +/** + * Pure normalization from raw LLM operations to repository-ready + * operations. The LLM output already passed zod validation; this layer + * enforces the invariants the schema cannot express: + * - merge/deprecate must target an existing active item of the same kind + * (otherwise downgraded to skip — conservative, never fabricates) + * - entity knowhereDocumentIds are intersected with the document ids + * actually referenced in the turn (the model cannot invent provenance) + * - create ignores any targetItemId the model may have emitted + * - create/merge payloads must yield at least one lexical token, otherwise + * the item could never be retrieved for later dedup + * - merge unions aliases (and entity document ids) with the target so + * prior search terms / provenance are not wiped by a partial rewrite + */ + +export type ResolvedMemoryOperation = + | { + readonly op: "create" + readonly kind: FluidMemoryKind + readonly payload: FluidMemoryPayload + readonly abstractL0: string + readonly overviewL1: string + readonly confidence: number + readonly summary: string + readonly reason?: string + } + | { + readonly op: "skip" + readonly kind: FluidMemoryKind + readonly summary: string + readonly reason?: string + } + | { + readonly op: "merge" + readonly kind: FluidMemoryKind + readonly targetItemId: string + readonly payload: FluidMemoryPayload + readonly abstractL0: string + readonly overviewL1: string + readonly confidence: number + readonly summary: string + readonly reason?: string + } + | { + readonly op: "deprecate" + readonly kind: FluidMemoryKind + readonly targetItemId: string + readonly summary: string + readonly reason?: string + } + +export type ExistingMemoryItemRef = { + readonly id: string + readonly kind: string + readonly status: string + readonly payload?: unknown +} + +type CandidateEntry = { + readonly abstractL0: string + readonly overviewL1: string + readonly confidence: number + readonly decision: { + readonly op: "create" | "skip" | "merge" | "deprecate" + readonly targetItemId?: string + readonly reason?: string + } +} + +const kindToArrayKey = { + indicator_pref: "indicatorPrefs", + stance: "stances", + decision_rule: "decisionRules", + entity_of_interest: "entities", +} as const + +export function resolveMemoryOperations(input: { + readonly operations: MemoryOperations + readonly existingItems: readonly ExistingMemoryItemRef[] + readonly referencedDocumentIds: readonly string[] +}): ResolvedMemoryOperation[] { + const activeById = new Map( + input.existingItems + .filter((item) => item.status === "active") + .map((item) => [item.id, item] as const), + ) + const allowedDocumentIds = new Set(input.referencedDocumentIds) + + const resolved: ResolvedMemoryOperation[] = [] + + for (const kind of Object.keys(kindToArrayKey) as FluidMemoryKind[]) { + const entries = input.operations[kindToArrayKey[kind]] as readonly (CandidateEntry & + Record)[] + + for (const entry of entries) { + const summary = entry.abstractL0 + const reason = entry.decision.reason + + if (entry.decision.op === "skip") { + resolved.push({ op: "skip", kind, summary, ...(reason ? { reason } : {}) }) + continue + } + + if (entry.decision.op === "merge" || entry.decision.op === "deprecate") { + const targetId = entry.decision.targetItemId + const target = targetId ? activeById.get(targetId) : undefined + if (!target || target.kind !== kind) { + resolved.push({ + op: "skip", + kind, + summary, + reason: `${entry.decision.op} target missing, inactive, or kind mismatch`, + }) + continue + } + if (entry.decision.op === "deprecate") { + resolved.push({ + op: "deprecate", + kind, + targetItemId: target.id, + summary, + ...(reason ? { reason } : {}), + }) + continue + } + const mergePayload = toPayload(kind, entry, allowedDocumentIds) + if (!mergePayload) { + resolved.push({ + op: "skip", + kind, + summary, + reason: "merged payload failed validation", + }) + continue + } + const preserved = preserveFieldsOnMerge( + kind, + mergePayload, + target.payload, + ) + if (!isIndexable(kind, preserved)) { + resolved.push({ + op: "skip", + kind, + summary, + reason: "merged payload has no searchable tokens", + }) + continue + } + resolved.push({ + op: "merge", + kind, + targetItemId: target.id, + payload: preserved, + abstractL0: entry.abstractL0, + overviewL1: entry.overviewL1, + confidence: entry.confidence, + summary, + ...(reason ? { reason } : {}), + }) + continue + } + + const createPayload = toPayload(kind, entry, allowedDocumentIds) + if (!createPayload) { + resolved.push({ + op: "skip", + kind, + summary, + reason: "payload failed validation", + }) + continue + } + if (!isIndexable(kind, createPayload)) { + resolved.push({ + op: "skip", + kind, + summary, + reason: "payload has no searchable tokens", + }) + continue + } + resolved.push({ + op: "create", + kind, + payload: createPayload, + abstractL0: entry.abstractL0, + overviewL1: entry.overviewL1, + confidence: entry.confidence, + summary, + ...(reason ? { reason } : {}), + }) + } + } + + return resolved +} + +function toPayload( + kind: FluidMemoryKind, + entry: Record, + allowedDocumentIds: ReadonlySet, +): FluidMemoryPayload | null { + const candidate: Record = { + name: entry.name, + aliases: entry.aliases, + definition: entry.definition, + polarity: entry.polarity, + importance: entry.importance, + formulaHint: entry.formulaHint, + statement: entry.statement, + scope: entry.scope, + rationale: entry.rationale, + when: entry.when, + then: entry.then, + priority: entry.priority, + ticker: entry.ticker, + reason: entry.reason, + knowhereDocumentIds: Array.isArray(entry.knowhereDocumentIds) + ? entry.knowhereDocumentIds.filter( + (id): id is string => + typeof id === "string" && allowedDocumentIds.has(id), + ) + : [], + } + return parseFluidMemoryPayload(kind, candidate) +} + +/** Reject payloads that could never be found again by the token index. */ +function isIndexable(kind: FluidMemoryKind, payload: FluidMemoryPayload): boolean { + return buildMemoryItemTokens(kind, payload).length > 0 +} + +/** + * Merge replaces the stored payload, but the model only sees this turn. + * Union aliases (and entity document ids) with the target so earlier search + * terms / provenance survive a partial rewrite. + */ +function preserveFieldsOnMerge( + kind: FluidMemoryKind, + mergedPayload: FluidMemoryPayload, + existingPayload: unknown, +): FluidMemoryPayload { + const existing = parseFluidMemoryPayload(kind, existingPayload) + if (!existing) return mergedPayload + + if ( + kind === "indicator_pref" && + "aliases" in mergedPayload && + "aliases" in existing + ) { + return { + ...mergedPayload, + aliases: unionStrings(existing.aliases, mergedPayload.aliases), + } + } + + if ( + kind === "entity_of_interest" && + "aliases" in mergedPayload && + "aliases" in existing && + "knowhereDocumentIds" in mergedPayload && + "knowhereDocumentIds" in existing + ) { + return { + ...mergedPayload, + aliases: unionStrings(existing.aliases, mergedPayload.aliases), + knowhereDocumentIds: unionStrings( + existing.knowhereDocumentIds, + mergedPayload.knowhereDocumentIds, + ), + } + } + + return mergedPayload +} + +function unionStrings( + left: readonly string[], + right: readonly string[], +): string[] { + return [...new Set([...left, ...right])] +} + +/** Diff-audit view of a resolved operation (itemId filled after write). */ +export function toDiffOperation( + operation: ResolvedMemoryOperation, + itemId?: string, +): MemoryDiffOperation { + const base = { + kind: operation.kind, + summary: operation.summary, + ...(operation.reason ? { reason: operation.reason } : {}), + } + switch (operation.op) { + case "create": + return { op: "create", ...base, ...(itemId ? { itemId } : {}) } + case "merge": + return { op: "merge", ...base, itemId: operation.targetItemId } + case "deprecate": + return { op: "deprecate", ...base, itemId: operation.targetItemId } + case "skip": + return { op: "skip", ...base } + } +} diff --git a/src/domains/memory/search-index.test.ts b/src/domains/memory/search-index.test.ts new file mode 100644 index 00000000..cbf5d0b3 --- /dev/null +++ b/src/domains/memory/search-index.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from "vitest" + +import { + buildMemoryItemTokens, + buildMemorySearchText, + tokenizeMemoryText, +} from "./search-index" + +describe("tokenizeMemoryText", () => { + it("splits CJK into single characters and latin into words", () => { + expect(tokenizeMemoryText("毛利率 PE gross_margin")).toEqual([ + { token: "毛", frequency: 1 }, + { token: "利", frequency: 1 }, + { token: "率", frequency: 1 }, + { token: "pe", frequency: 1 }, + { token: "gross_margin", frequency: 1 }, + ]) + }) + + it("counts repeated tokens", () => { + expect(tokenizeMemoryText("PE pe Pe")).toEqual([ + { token: "pe", frequency: 3 }, + ]) + }) + + it("returns empty for whitespace-only input", () => { + expect(tokenizeMemoryText(" ")).toEqual([]) + }) +}) + +describe("buildMemorySearchText", () => { + it("indexes indicator name, aliases, and definition", () => { + expect( + buildMemorySearchText("indicator_pref", { + name: "毛利率", + aliases: ["gross margin"], + definition: "毛利除以营收", + polarity: "higher_better", + importance: "core", + }), + ).toBe("毛利率 gross margin 毛利除以营收") + }) + + it("indexes entity name, aliases, and ticker without reason", () => { + expect( + buildMemorySearchText("entity_of_interest", { + name: "英伟达", + aliases: ["NVIDIA"], + ticker: "NVDA", + knowhereDocumentIds: ["doc-1"], + reason: "一直在跟踪", + }), + ).toBe("英伟达 NVIDIA NVDA") + }) + + it("indexes stance statement and scope", () => { + expect( + buildMemorySearchText("stance", { + statement: "做长期投资", + scope: "宏观短期观点", + rationale: "美联储短期说法不重要", + }), + ).toBe("做长期投资 宏观短期观点") + }) + + it("indexes decision rule when and then", () => { + expect( + buildMemorySearchText("decision_rule", { + when: "毛利率连续两季下滑", + then: "减仓观望", + priority: "high", + rationale: "用户明确说过", + }), + ).toBe("毛利率连续两季下滑 减仓观望") + }) +}) + +describe("buildMemoryItemTokens", () => { + it("tokenizes the search text of an item", () => { + const tokens = buildMemoryItemTokens("indicator_pref", { + name: "PE", + aliases: [], + definition: "市盈率", + polarity: "context", + importance: "secondary", + }) + expect(tokens.map((token) => token.token)).toEqual([ + "pe", + "市", + "盈", + "率", + ]) + }) +}) diff --git a/src/domains/memory/search-index.ts b/src/domains/memory/search-index.ts new file mode 100644 index 00000000..84a53e9b --- /dev/null +++ b/src/domains/memory/search-index.ts @@ -0,0 +1,84 @@ +import type { + DecisionRulePayload, + EntityOfInterestPayload, + FluidMemoryKind, + FluidMemoryPayload, + IndicatorPreferencePayload, + StancePayload, +} from "./types" + +/** + * Lexical search-index helpers for fluid memory dedup retrieval. + * + * Pure and dependency-free so both the write path (indexing an item) and the + * read path (turning a turn into a query) share one tokenizer. Tokenization + * mirrors Knowhere map-nav: lowercase, then emit single CJK characters and + * `[a-z0-9_]+` runs. This handles Chinese (no whitespace segmentation) and + * Latin/alphanumeric terms without any Postgres extension. + */ + +export type MemoryToken = { + readonly token: string + readonly frequency: number +} + +// Single CJK char OR a run of latin letters / digits / underscore. +const TOKEN_PATTERN = + /[a-z0-9_]+|[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/g + +/** + * Build the text that represents an item for lexical matching. Only the + * fields a user would phrase a query against are included (names, aliases, + * short definitions), not provenance or bookkeeping fields. + */ +export function buildMemorySearchText( + kind: FluidMemoryKind, + payload: FluidMemoryPayload, +): string { + return collectSearchParts(kind, payload) + .filter((part) => part.length > 0) + .join(" ") +} + +/** Tokenize free text into deduped tokens with occurrence counts. */ +export function tokenizeMemoryText(text: string): MemoryToken[] { + const counts = new Map() + const matches = text.toLowerCase().match(TOKEN_PATTERN) + if (!matches) return [] + for (const token of matches) { + counts.set(token, (counts.get(token) ?? 0) + 1) + } + return [...counts].map(([token, frequency]) => ({ token, frequency })) +} + +/** Tokens that index one memory item (search text of its payload). */ +export function buildMemoryItemTokens( + kind: FluidMemoryKind, + payload: FluidMemoryPayload, +): MemoryToken[] { + return tokenizeMemoryText(buildMemorySearchText(kind, payload)) +} + +function collectSearchParts( + kind: FluidMemoryKind, + payload: FluidMemoryPayload, +): readonly string[] { + switch (kind) { + case "indicator_pref": { + const p = payload as IndicatorPreferencePayload + return [p.name, ...p.aliases, p.definition] + } + case "stance": { + const p = payload as StancePayload + return [p.statement, p.scope] + } + case "decision_rule": { + const p = payload as DecisionRulePayload + return [p.when, p.then] + } + case "entity_of_interest": { + const p = payload as EntityOfInterestPayload + return [p.name, ...p.aliases, ...(p.ticker ? [p.ticker] : [])] + } + } +} diff --git a/src/domains/memory/service.ts b/src/domains/memory/service.ts new file mode 100644 index 00000000..da772f65 --- /dev/null +++ b/src/domains/memory/service.ts @@ -0,0 +1,54 @@ +import "server-only" + +import { databaseRuntime } from "@/domains/workspace/database-runtime" +import { memoryRepository } from "./repository" +import type { ResolvedMemoryOperation } from "./resolve-operations" +import type { FluidMemoryKind, MemoryDiffOperation } from "./types" +import type { FluidMemoryItem } from "@/infrastructure/db/schema" + +type MemoryService = { + readonly findDedupCandidates: ( + workspaceId: string, + kind: FluidMemoryKind, + tokens: readonly string[], + limit: number, + ) => Promise + readonly applyOperations: ( + workspaceId: string, + sourceMessageId: string | null, + operations: readonly ResolvedMemoryOperation[], + ) => Promise +} + +const findDedupCandidates: MemoryService["findDedupCandidates"] = ( + workspaceId, + kind, + tokens, + limit, +) => + databaseRuntime.runPromise( + memoryRepository.findDedupCandidatesEffect( + workspaceId, + kind, + tokens, + limit, + ), + ) + +const applyOperations: MemoryService["applyOperations"] = ( + workspaceId, + sourceMessageId, + operations, +) => + databaseRuntime.runPromise( + memoryRepository.applyOperationsEffect( + workspaceId, + sourceMessageId, + operations, + ), + ) + +export const memoryService: MemoryService = { + findDedupCandidates, + applyOperations, +} diff --git a/src/domains/memory/types.ts b/src/domains/memory/types.ts new file mode 100644 index 00000000..dac7602d --- /dev/null +++ b/src/domains/memory/types.ts @@ -0,0 +1,101 @@ +import { z } from "zod" + +/** + * Fluid memory type contract. + * + * Four typed payload kinds, extracted from conversation turns. The DB + * stores `payload` as jsonb; these schemas are the validation boundary on + * both write (LLM output) and read (repository decode) paths. + */ + +export const fluidMemoryKinds = [ + "indicator_pref", + "stance", + "decision_rule", + "entity_of_interest", +] as const + +export type FluidMemoryKind = (typeof fluidMemoryKinds)[number] + +export const indicatorPreferencePayloadSchema = z.object({ + name: z.string().min(1), + aliases: z.array(z.string()).default([]), + definition: z.string().min(1), + polarity: z.enum(["higher_better", "lower_better", "context"]), + importance: z.enum(["core", "secondary"]), + formulaHint: z.preprocess( + (value) => (value === null ? undefined : value), + z.string().optional(), + ), +}) + +export const stancePayloadSchema = z.object({ + statement: z.string().min(1), + scope: z.string().min(1), + rationale: z.string().min(1), +}) + +export const decisionRulePayloadSchema = z.object({ + when: z.string().min(1), + then: z.string().min(1), + priority: z.enum(["high", "medium", "low"]), + rationale: z.string().min(1), +}) + +export const entityOfInterestPayloadSchema = z.object({ + name: z.string().min(1), + ticker: z.preprocess( + (value) => (value === null ? undefined : value), + z.string().optional(), + ), + aliases: z.array(z.string()).default([]), + knowhereDocumentIds: z.array(z.string()).default([]), + reason: z.string().min(1), +}) + +export type IndicatorPreferencePayload = z.infer< + typeof indicatorPreferencePayloadSchema +> +export type StancePayload = z.infer +export type DecisionRulePayload = z.infer +export type EntityOfInterestPayload = z.infer< + typeof entityOfInterestPayloadSchema +> + +export type FluidMemoryPayload = + | IndicatorPreferencePayload + | StancePayload + | DecisionRulePayload + | EntityOfInterestPayload + +const payloadSchemas: Record> = { + indicator_pref: indicatorPreferencePayloadSchema, + stance: stancePayloadSchema, + decision_rule: decisionRulePayloadSchema, + entity_of_interest: entityOfInterestPayloadSchema, +} + +export function isFluidMemoryKind(value: unknown): value is FluidMemoryKind { + return ( + typeof value === "string" && + (fluidMemoryKinds as readonly string[]).includes(value) + ) +} + +/** Decode a persisted jsonb payload; returns null when the row is malformed. */ +export function parseFluidMemoryPayload( + kind: FluidMemoryKind, + value: unknown, +): FluidMemoryPayload | null { + const result = payloadSchemas[kind].safeParse(value) + return result.success ? result.data : null +} + +/** One decided operation over the memory set; persisted into memory_diffs. */ +export type MemoryDiffOperation = { + readonly op: "create" | "skip" | "merge" | "deprecate" + readonly kind: FluidMemoryKind + readonly itemId?: string + readonly summary: string + readonly reason?: string +} diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index 4e9470f3..a3c98054 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -1,7 +1,9 @@ import { sql } from "drizzle-orm"; import { bigint, + doublePrecision, index, + integer, jsonb, pgTable, text, @@ -338,3 +340,139 @@ export const chatMessages = pgTable( export type ChatMessage = typeof chatMessages.$inferSelect; export type NewChatMessage = typeof chatMessages.$inferInsert; + +/** + * Fluid memory: typed insights extracted from human-AI conversation turns + * (as opposed to "crystal memory", which is the parsed document knowledge + * that stays upstream in Knowhere). + * + * One row per extracted insight. `kind` discriminates the typed `payload` + * (see src/domains/memory/types.ts for the payload contract per kind): + * - indicator_pref — a metric the user cares about (name, aliases, + * polarity, importance) + * - stance — a stated position that shapes judgement + * - decision_rule — a when/then rule over indicators + * - entity_of_interest — a company/topic the user tracks + * + * `abstract_l0` / `overview_l1` are the tiered sidecar summaries (L0 = + * one line for pre-filter/dedup context, L1 = short paragraph for later + * cognition injection). L2 is the payload itself. + * + * Lifecycle: rows start `active`; user revisions deprecate rather than + * delete (conservative merge policy), with `version` bumped on merge. + * + * `source_message_id` points at the assistant message of the turn the + * insight was extracted from; it is set-null on message deletion because + * the insight outlives any single turn. + */ +export const fluidMemoryItems = pgTable( + "fluid_memory_items", + { + id: uuid("id").primaryKey().defaultRandom(), + workspaceId: uuid("workspace_id") + .notNull() + .references(() => workspaces.id, { onDelete: "cascade" }), + kind: text("kind").notNull(), + payload: jsonb("payload").notNull(), + abstractL0: text("abstract_l0").notNull(), + overviewL1: text("overview_l1").notNull(), + sourceMessageId: uuid("source_message_id").references( + () => chatMessages.id, + { onDelete: "set null" }, + ), + confidence: doublePrecision("confidence").notNull(), + status: text("status").notNull(), + version: integer("version").notNull().default(1), + createdAt: timestamp("created_at", { withTimezone: true }) + .notNull() + .defaultNow(), + updatedAt: timestamp("updated_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (t) => [ + // Workspace lifecycle scans (active vs deprecated). + index("fluid_memory_items_workspace_status_idx").on( + t.workspaceId, + t.status, + ), + index("fluid_memory_items_workspace_kind_idx").on(t.workspaceId, t.kind), + ], +); + +export type FluidMemoryItem = typeof fluidMemoryItems.$inferSelect; +export type NewFluidMemoryItem = typeof fluidMemoryItems.$inferInsert; + +/** + * Lexical inverted index over active fluid memory items, used to retrieve + * dedup candidates at extraction time instead of loading the whole memory + * set into the prompt. One row per (item, token); `frequency` counts token + * occurrences in the item's search text. + * + * Invariant: token rows exist iff the owning item is `active`. Writers keep + * this in sync — create inserts rows, merge replaces them, deprecate deletes + * them — so lookups scan tokens alone (no status join) and never surface a + * deprecated item. + * + * Tokenization mirrors Knowhere map-nav: single CJK characters plus + * `[a-z0-9_]+` runs. Scoring is idf-weighted token overlap computed in SQL, + * keeping the mechanism on portable Postgres (no pg_trgm/pgvector). + */ +export const fluidMemoryTokens = pgTable( + "fluid_memory_tokens", + { + id: uuid("id").primaryKey().defaultRandom(), + workspaceId: uuid("workspace_id") + .notNull() + .references(() => workspaces.id, { onDelete: "cascade" }), + itemId: uuid("item_id") + .notNull() + .references(() => fluidMemoryItems.id, { onDelete: "cascade" }), + kind: text("kind").notNull(), + token: text("token").notNull(), + frequency: integer("frequency").notNull().default(1), + }, + (t) => [ + // Lookup: candidate tokens within a workspace + kind scope. + index("fluid_memory_tokens_lookup_idx").on( + t.workspaceId, + t.kind, + t.token, + ), + // Rebuild/delete a single item's rows on merge/deprecate. + index("fluid_memory_tokens_item_idx").on(t.itemId), + ], +); + +export type FluidMemoryToken = typeof fluidMemoryTokens.$inferSelect; +export type NewFluidMemoryToken = typeof fluidMemoryTokens.$inferInsert; + +/** + * Append-only audit of extraction decisions, one row per processed turn. + * `operations` is a JSONB array of { op, kind, itemId?, summary, reason? } + * records (op = create | skip | merge | deprecate), mirroring OpenViking's + * memory_diff.json so memory growth stays observable and reversible. + */ +export const memoryDiffs = pgTable( + "memory_diffs", + { + id: uuid("id").primaryKey().defaultRandom(), + workspaceId: uuid("workspace_id") + .notNull() + .references(() => workspaces.id, { onDelete: "cascade" }), + sourceMessageId: uuid("source_message_id").references( + () => chatMessages.id, + { onDelete: "set null" }, + ), + operations: jsonb("operations").notNull(), + createdAt: timestamp("created_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (t) => [ + index("memory_diffs_workspace_created_idx").on(t.workspaceId, t.createdAt), + ], +); + +export type MemoryDiff = typeof memoryDiffs.$inferSelect; +export type NewMemoryDiff = typeof memoryDiffs.$inferInsert; From 1d76c7254cb94fd51d9dd2a9f136fbb703352212 Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Wed, 2 Sep 2026 11:03:27 +0800 Subject: [PATCH 04/11] fix: stop feeding duplicate evidenceText into search tool context Notebook agent grounding already comes from result chunks with citeable refs; injecting evidenceText repeated the same bodies without refs. Co-authored-by: Cursor --- src/agent-harness/knowhere-text.test.ts | 3 +++ src/agent-harness/knowhere-text.ts | 3 ++- src/domains/chat/index.ts | 12 ------------ src/domains/chat/media-assets.test.ts | 2 -- src/domains/chat/media-assets.ts | 1 - 5 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/agent-harness/knowhere-text.test.ts b/src/agent-harness/knowhere-text.test.ts index e916cad2..5ec8abbb 100644 --- a/src/agent-harness/knowhere-text.test.ts +++ b/src/agent-harness/knowhere-text.test.ts @@ -25,8 +25,11 @@ describe("knowhereToolText", () => { expect(text).toContain('') expect(text).toContain('ref="r1:result:1"') expect(text).toContain('ref="asset:r1:result:1"') + expect(text).toContain("Page one summary.") expect(text).toContain("Call inspectImage") expect(text).toContain("before finalize") + expect(text).not.toContain("") + expect(text).not.toContain("Page one evidence.") expect(text).not.toContain("https://assets.example/page-1.png") }) diff --git a/src/agent-harness/knowhere-text.ts b/src/agent-harness/knowhere-text.ts index f5b19da3..2140e7ec 100644 --- a/src/agent-harness/knowhere-text.ts +++ b/src/agent-harness/knowhere-text.ts @@ -56,7 +56,8 @@ export const knowhereToolText = { stopReason: input.response.stopReason ?? undefined, failureReason: input.response.failureReason ?? undefined, }), - formatOptionalTextTag("evidence", input.response.evidenceText), + // Model grounding comes from (results). Do not also inject + // evidenceText — same bodies, no citeable refs, doubles context. formatEvidenceChunks(input.chunks), formatEvidenceAssets(input.assets), formatAssetInstruction(input.assets), diff --git a/src/domains/chat/index.ts b/src/domains/chat/index.ts index d2b7208f..11a7b85c 100644 --- a/src/domains/chat/index.ts +++ b/src/domains/chat/index.ts @@ -291,7 +291,6 @@ export const answerQuestionWithRetrieval = ( results: useNotebookSourceTitles(rawResults, input.sources), sources: input.sources, hardenChatAssetUrl: input.hardenChatAssetUrl, - evidenceText: formatRetrievalEvidenceText(retrievalResponses), }), ) const pageCitationResults = yield* Effect.tryPromise(() => @@ -1343,17 +1342,6 @@ function hasDisplayedManifestArtifacts(result: HarnessRunResult): boolean { return result.manifest.artifacts.some((artifact) => artifact.display) } -function formatRetrievalEvidenceText( - responses: readonly RetrievalQueryResponse[], -): string | undefined { - const evidenceText = responses - .map((response): string => response.evidenceText?.trim() ?? "") - .filter((value): boolean => value.length > 0) - .join("\n") - - return evidenceText || undefined -} - function getRetrievalResultKey(result: RetrievalResult): string { const source = result.source return [ diff --git a/src/domains/chat/media-assets.test.ts b/src/domains/chat/media-assets.test.ts index 470255df..1d744826 100644 --- a/src/domains/chat/media-assets.test.ts +++ b/src/domains/chat/media-assets.test.ts @@ -103,8 +103,6 @@ describe("chat media assets", () => { }), ], hardenChatAssetUrl, - evidenceText: - "[image-6-中华人民共和国居民身份证.jpg]\n[image-7-中国居民身份证.jpg]", }) expect(results).toHaveLength(1) diff --git a/src/domains/chat/media-assets.ts b/src/domains/chat/media-assets.ts index bafaae88..423a8788 100644 --- a/src/domains/chat/media-assets.ts +++ b/src/domains/chat/media-assets.ts @@ -27,7 +27,6 @@ export type RetrievalResultAssetInput = { readonly results: readonly RetrievalResult[] readonly sources: readonly Source[] readonly hardenChatAssetUrl?: HardenChatAssetUrl - readonly evidenceText?: string } export async function enrichRetrievalResultsWithAssetUrls({ From 15d5cdeee5e0af280cbb1a20a3d3c2bcd8e6f9da Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Wed, 2 Sep 2026 11:07:00 +0800 Subject: [PATCH 05/11] test: assert search tool grounding comes from chunks only Co-authored-by: Cursor --- src/agent-harness/runtime.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/agent-harness/runtime.test.ts b/src/agent-harness/runtime.test.ts index 0a5ca193..74e7ef71 100644 --- a/src/agent-harness/runtime.test.ts +++ b/src/agent-harness/runtime.test.ts @@ -156,7 +156,8 @@ describe("agent harness runtime", () => { expect(firstResult).toContain('ref="r1:result:1"') expect(secondResult).toContain('retrievalCount="2"') - expect(secondResult).toContain("Second evidence") + expect(secondResult).toContain("Second retrieval evidence.") + expect(secondResult).not.toContain("") expect(secondResult).toContain('ref="r2:result:1"') expect(JSON.stringify(secondResult)).not.toContain("r1:result:1") expect(ledger.snapshot().chunks.map((chunk) => chunk.ref)).toEqual([ From 02151b2da9a0814223ec62127a92350a8631f8a9 Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Wed, 9 Sep 2026 14:45:48 +0800 Subject: [PATCH 06/11] feat(memory): implement coarse capture of user observations - Introduced a new observation layer for capturing user insights without early classification. - Updated the memory extraction workflow to persist observations and trigger distillation. - Refactored related services and repository methods to support the new observation model. - Added tests for the new capture prompt and ensured existing functionality remains intact. This change enhances the ability to gather user concerns and insights for future processing. --- .pnpm-store/v11/index.db | Bin 0 -> 8192 bytes .repos/OpenViking | 1 + drizzle/0015_bumpy_vulcan.sql | 17 + drizzle/meta/0015_snapshot.json | 1444 +++++++++++++++++ drizzle/meta/_journal.json | 7 + pnpm-workspace.yaml | 8 + src/app/api/memory/distill/route.ts | 27 + src/domains/memory/distill-config.ts | 19 + src/domains/memory/distill-model.ts | 83 + src/domains/memory/distill-prompts.test.ts | 110 ++ src/domains/memory/distill-prompts.ts | 311 ++++ src/domains/memory/distill-trigger.test.ts | 84 + src/domains/memory/distill-trigger.ts | 77 + src/domains/memory/distill-types.test.ts | 209 +++ src/domains/memory/distill-types.ts | 190 +++ src/domains/memory/distill-workflow.test.ts | 48 + src/domains/memory/distill-workflow.ts | 231 +++ src/domains/memory/extract-workflow.test.ts | 40 + src/domains/memory/extract-workflow.ts | 87 +- src/domains/memory/extraction-model.ts | 33 +- src/domains/memory/observation-types.test.ts | 78 + src/domains/memory/observation-types.ts | 50 + src/domains/memory/prompts.test.ts | 33 +- src/domains/memory/prompts.ts | 279 +--- src/domains/memory/repository.ts | 428 +++-- src/domains/memory/resolve-operations.test.ts | 2 +- src/domains/memory/resolve-operations.ts | 36 +- src/domains/memory/service.ts | 66 +- src/infrastructure/db/schema.ts | 50 + 29 files changed, 3562 insertions(+), 486 deletions(-) create mode 100644 .pnpm-store/v11/index.db create mode 160000 .repos/OpenViking create mode 100644 drizzle/0015_bumpy_vulcan.sql create mode 100644 drizzle/meta/0015_snapshot.json create mode 100644 src/app/api/memory/distill/route.ts create mode 100644 src/domains/memory/distill-config.ts create mode 100644 src/domains/memory/distill-model.ts create mode 100644 src/domains/memory/distill-prompts.test.ts create mode 100644 src/domains/memory/distill-prompts.ts create mode 100644 src/domains/memory/distill-trigger.test.ts create mode 100644 src/domains/memory/distill-trigger.ts create mode 100644 src/domains/memory/distill-types.test.ts create mode 100644 src/domains/memory/distill-types.ts create mode 100644 src/domains/memory/distill-workflow.test.ts create mode 100644 src/domains/memory/distill-workflow.ts create mode 100644 src/domains/memory/extract-workflow.test.ts create mode 100644 src/domains/memory/observation-types.test.ts create mode 100644 src/domains/memory/observation-types.ts diff --git a/.pnpm-store/v11/index.db b/.pnpm-store/v11/index.db new file mode 100644 index 0000000000000000000000000000000000000000..8fdf9e7d3d8db312b5d08efe43a80ea9046a773f GIT binary patch literal 8192 zcmeIuKa0XJ7zXe(90-L&2f@*I^A3(%?hDvzz+$vsP3gHPQKLxxhjh@v!PQS~E2W&1 zyF3p`-Xw39{D!}Bl^9y=4jY}&534ZFS(At{#`Bq$d#rSQ%lBEy&dc$*8rJ2U=;_+*^(@v)Gg)ot=J;^ntw1bDd~%rSDSTSJV%sESjGbd8*@wiUL3K k+WEbmsaIZ*9$$h01Rwwb2tWV=5P$##AOHafK%fw~0WYO9761SM literal 0 HcmV?d00001 diff --git a/.repos/OpenViking b/.repos/OpenViking new file mode 160000 index 00000000..f6d9dec6 --- /dev/null +++ b/.repos/OpenViking @@ -0,0 +1 @@ +Subproject commit f6d9dec6b6ae16a152c437fd4ad81ca45fcc8648 diff --git a/drizzle/0015_bumpy_vulcan.sql b/drizzle/0015_bumpy_vulcan.sql new file mode 100644 index 00000000..2b383759 --- /dev/null +++ b/drizzle/0015_bumpy_vulcan.sql @@ -0,0 +1,17 @@ +CREATE TABLE "fluid_observations" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "source_message_id" uuid, + "signal" text NOT NULL, + "evidence_quote" text NOT NULL, + "subject_hint" text, + "referenced_document_ids" jsonb DEFAULT '[]'::jsonb NOT NULL, + "confidence" double precision NOT NULL, + "status" text DEFAULT 'pending' NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "consumed_at" timestamp with time zone +); +--> statement-breakpoint +ALTER TABLE "fluid_observations" ADD CONSTRAINT "fluid_observations_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "fluid_observations" ADD CONSTRAINT "fluid_observations_source_message_id_chat_messages_id_fk" FOREIGN KEY ("source_message_id") REFERENCES "public"."chat_messages"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "fluid_observations_workspace_status_created_idx" ON "fluid_observations" USING btree ("workspace_id","status","created_at"); \ No newline at end of file diff --git a/drizzle/meta/0015_snapshot.json b/drizzle/meta/0015_snapshot.json new file mode 100644 index 00000000..3b313fd5 --- /dev/null +++ b/drizzle/meta/0015_snapshot.json @@ -0,0 +1,1444 @@ +{ + "id": "be8deb1f-3ec8-4304-85d0-41403629970c", + "prevId": "72bfb4fe-4048-4dce-a5fa-ce8eea880aa3", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_memory_items": { + "name": "fluid_memory_items", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "abstract_l0": { + "name": "abstract_l0", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "overview_l1": { + "name": "overview_l1", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "fluid_memory_items_workspace_status_idx": { + "name": "fluid_memory_items_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fluid_memory_items_workspace_kind_idx": { + "name": "fluid_memory_items_workspace_kind_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_memory_items_workspace_id_workspaces_id_fk": { + "name": "fluid_memory_items_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_memory_items_source_message_id_chat_messages_id_fk": { + "name": "fluid_memory_items_source_message_id_chat_messages_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_memory_tokens": { + "name": "fluid_memory_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "item_id": { + "name": "item_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "frequency": { + "name": "frequency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + } + }, + "indexes": { + "fluid_memory_tokens_lookup_idx": { + "name": "fluid_memory_tokens_lookup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fluid_memory_tokens_item_idx": { + "name": "fluid_memory_tokens_item_idx", + "columns": [ + { + "expression": "item_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_memory_tokens_workspace_id_workspaces_id_fk": { + "name": "fluid_memory_tokens_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_memory_tokens", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_memory_tokens_item_id_fluid_memory_items_id_fk": { + "name": "fluid_memory_tokens_item_id_fluid_memory_items_id_fk", + "tableFrom": "fluid_memory_tokens", + "tableTo": "fluid_memory_items", + "columnsFrom": [ + "item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_observations": { + "name": "fluid_observations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "signal": { + "name": "signal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "evidence_quote": { + "name": "evidence_quote", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "subject_hint": { + "name": "subject_hint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "referenced_document_ids": { + "name": "referenced_document_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "confidence": { + "name": "confidence", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "fluid_observations_workspace_status_created_idx": { + "name": "fluid_observations_workspace_status_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_observations_workspace_id_workspaces_id_fk": { + "name": "fluid_observations_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_observations", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_observations_source_message_id_chat_messages_id_fk": { + "name": "fluid_observations_source_message_id_chat_messages_id_fk", + "tableFrom": "fluid_observations", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_diffs": { + "name": "memory_diffs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memory_diffs_workspace_created_idx": { + "name": "memory_diffs_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_diffs_workspace_id_workspaces_id_fk": { + "name": "memory_diffs_workspace_id_workspaces_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memory_diffs_source_message_id_chat_messages_id_fk": { + "name": "memory_diffs_source_message_id_chat_messages_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parsed_document_sync_leases": { + "name": "parsed_document_sync_leases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_token": { + "name": "lease_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_reason": { + "name": "release_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parsed_document_sync_leases_token_idx": { + "name": "parsed_document_sync_leases_token_idx", + "columns": [ + { + "expression": "lease_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_active_idx": { + "name": "parsed_document_sync_leases_active_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_workspace_active_idx": { + "name": "parsed_document_sync_leases_workspace_active_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_document_active_idx": { + "name": "parsed_document_sync_leases_document_active_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parsed_document_sync_leases_workspace_id_workspaces_id_fk": { + "name": "parsed_document_sync_leases_workspace_id_workspaces_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parsed_document_sync_leases_source_id_sources_id_fk": { + "name": "parsed_document_sync_leases_source_id_sources_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index b9be4f54..32ea955d 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -106,6 +106,13 @@ "when": 1788252035435, "tag": "0014_messy_apocalypse", "breakpoints": true + }, + { + "idx": 15, + "version": "7", + "when": 1788418416499, + "tag": "0015_bumpy_vulcan", + "breakpoints": true } ] } \ No newline at end of file diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 80ee5bbc..082ea2dc 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,3 +1,11 @@ +allowBuilds: + core-js: set this to true or false + esbuild: set this to true or false + msgpackr-extract: set this to true or false + msw: set this to true or false + sharp: set this to true or false + unrs-resolver: set this to true or false + ignoredBuiltDependencies: - sharp - unrs-resolver diff --git a/src/app/api/memory/distill/route.ts b/src/app/api/memory/distill/route.ts new file mode 100644 index 00000000..e0230d93 --- /dev/null +++ b/src/app/api/memory/distill/route.ts @@ -0,0 +1,27 @@ +import { serve } from "@upstash/workflow/nextjs" + +import { + normalizeMemoryDistillPayload, + runMemoryDistillWorkflow, + type MemoryDistillPayload, +} from "@/domains/memory/distill-workflow" +import { logger } from "@/lib/logger" + +export const { POST } = serve( + async (context) => { + const payload = normalizeMemoryDistillPayload(context.requestPayload) + if (!payload) { + logger.warn("memory: distill workflow received invalid payload") + return + } + await runMemoryDistillWorkflow({ context, payload }) + }, + { + failureFunction: async ({ context, failResponse }) => { + logger.error("memory: distill workflow failed", { + payload: context.requestPayload, + failResponse, + }) + }, + }, +) diff --git a/src/domains/memory/distill-config.ts b/src/domains/memory/distill-config.ts new file mode 100644 index 00000000..4c82abbf --- /dev/null +++ b/src/domains/memory/distill-config.ts @@ -0,0 +1,19 @@ +/** + * Distill job defaults from the two-tier fluid-memory plan. + * Tunable later; kept as named constants (not scattered literals). + */ + +/** Process-local cooldown + QStash workflowRunId bucket width (mirrors reconcile). */ +export const DISTILL_COOLDOWN_MS = 5 * 60_000 + +/** Capture may trigger distill once pending observations reach this count. */ +export const DISTILL_MIN_PENDING = 8 + +/** Max pending rows claimed per distill run (oldest first). */ +export const DISTILL_BATCH_MAX = 40 + +/** Lexical dedup candidates loaded per memory kind for one distill batch. */ +export const DISTILL_DEDUP_CANDIDATES_PER_KIND = 8 + +/** Delete consumed observations older than this (retention sweep after distill). */ +export const DISTILL_CONSUMED_RETENTION_MS = 30 * 24 * 60_000 diff --git a/src/domains/memory/distill-model.ts b/src/domains/memory/distill-model.ts new file mode 100644 index 00000000..ef2485ff --- /dev/null +++ b/src/domains/memory/distill-model.ts @@ -0,0 +1,83 @@ +import "server-only" + +import { generateObject } from "ai" + +import { buildDistillPrompt } from "./distill-prompts" +import { + entityDistillOutputSchema, + experienceDistillOutputSchema, + indicatorDistillOutputSchema, + toMemoryOperations, + type DistillObservationInput, + type DistillPassKind, + type ExistingMemoryContextItem, + type MemoryOperations, +} from "./distill-types" +import { CHAT_MODEL } from "@/lib/ai" +import { summarizeUnknownError } from "@/lib/format-log-value" +import { logger } from "@/lib/logger" + +const MEMORY_EXTRACTION_MODEL = process.env.MEMORY_EXTRACTION_MODEL ?? CHAT_MODEL + +/** + * One structured-output call for a single distill pass. + * Best-effort — distill runs as a background job; a model failure returns + * null so the workflow can skip applying that pass (logged). No multi-level + * fallback chain. + * + * Input: pending observation batch + existing memories of kinds this pass may + * write + allowed document ids for the batch. + * Output: full MemoryOperations with only this pass's arrays populated + * (others empty), ready for resolveMemoryOperations. + */ +export async function distillMemoryPass(input: { + readonly pass: DistillPassKind + readonly workspaceId: string + readonly observations: readonly DistillObservationInput[] + readonly existingItems: readonly ExistingMemoryContextItem[] + readonly referencedDocumentIds: readonly string[] +}): Promise { + const prompt = buildDistillPrompt(input.pass, { + observations: input.observations, + existingItems: input.existingItems, + referencedDocumentIds: input.referencedDocumentIds, + }) + + try { + switch (input.pass) { + case "indicator": { + const response = await generateObject({ + model: MEMORY_EXTRACTION_MODEL, + schema: indicatorDistillOutputSchema, + messages: [{ role: "user", content: prompt }], + }) + return toMemoryOperations("indicator", response.object) + } + case "experience": { + const response = await generateObject({ + model: MEMORY_EXTRACTION_MODEL, + schema: experienceDistillOutputSchema, + messages: [{ role: "user", content: prompt }], + }) + return toMemoryOperations("experience", response.object) + } + case "entity": { + const response = await generateObject({ + model: MEMORY_EXTRACTION_MODEL, + schema: entityDistillOutputSchema, + messages: [{ role: "user", content: prompt }], + }) + return toMemoryOperations("entity", response.object) + } + } + } catch (error) { + logger.warn("memory: distill model call failed; skipping pass", { + workspaceId: input.workspaceId, + pass: input.pass, + model: MEMORY_EXTRACTION_MODEL, + observationCount: input.observations.length, + error: summarizeUnknownError(error), + }) + return null + } +} diff --git a/src/domains/memory/distill-prompts.test.ts b/src/domains/memory/distill-prompts.test.ts new file mode 100644 index 00000000..94578800 --- /dev/null +++ b/src/domains/memory/distill-prompts.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from "vitest" + +import { buildDistillPrompt } from "./distill-prompts" +import type { DistillObservationInput } from "./distill-types" + +const observations: DistillObservationInput[] = [ + { + id: "obs-1", + signal: "看重毛利率", + evidenceQuote: "毛利率是核心", + subjectHint: "毛利率", + confidence: 0.9, + referencedDocumentIds: ["doc-1"], + }, + { + id: "obs-2", + signal: "跟踪英伟达", + evidenceQuote: "英伟达一直在跟踪", + subjectHint: "英伟达", + confidence: 0.8, + referencedDocumentIds: [], + }, +] + +describe("buildDistillPrompt", () => { + it("indicator pass: only indicator schema, no other kind arrays", () => { + const prompt = buildDistillPrompt("indicator", { + observations, + existingItems: [ + { + id: "item-1", + kind: "indicator_pref", + abstractL0: "看重毛利率", + payloadSummary: "毛利率 — 毛利占营收", + }, + ], + referencedDocumentIds: ["doc-1"], + }) + + expect(prompt).toContain("You DISTILL indicator preferences") + expect(prompt).toContain('"indicatorPrefs"') + expect(prompt).not.toContain('"stances"') + expect(prompt).not.toContain('"decisionRules"') + expect(prompt).not.toContain('"entities"') + expect(prompt).toContain("never emit") + expect(prompt).toContain("PENDING OBSERVATIONS") + expect(prompt).toContain("id=obs-1") + expect(prompt).toContain("id=obs-2") + expect(prompt).toContain("看重毛利率") + expect(prompt).toContain("跟踪英伟达") + expect(prompt).toContain("id=item-1") + expect(prompt).toContain("doc-1") + // All observations go to every pass — no kind routing. + expect(prompt.indexOf("obs-1")).toBeLessThan(prompt.indexOf("obs-2")) + }) + + it("experience pass: stances+rules only; still sees full observation batch", () => { + const prompt = buildDistillPrompt("experience", { + observations, + existingItems: [], + referencedDocumentIds: [], + }) + + expect(prompt).toContain("You DISTILL stances and decision rules") + expect(prompt).toContain('"stances"') + expect(prompt).toContain('"decisionRules"') + expect(prompt).not.toContain('"indicatorPrefs"') + expect(prompt).not.toContain('"entities"') + expect(prompt).toContain("id=obs-1") + expect(prompt).toContain("id=obs-2") + expect(prompt).toContain("(no existing memories yet)") + expect(prompt).toContain("(no documents referenced in this batch)") + }) + + it("entity pass: entities only; referenced ids from batch", () => { + const prompt = buildDistillPrompt("entity", { + observations, + existingItems: [ + { + id: "item-4", + kind: "entity_of_interest", + abstractL0: "跟踪英伟达", + payloadSummary: "英伟达 NVDA", + }, + ], + referencedDocumentIds: ["doc-1"], + }) + + expect(prompt).toContain("You DISTILL entities of interest") + expect(prompt).toContain('"entities"') + expect(prompt).not.toContain('"indicatorPrefs"') + expect(prompt).not.toContain('"stances"') + expect(prompt).not.toContain('"decisionRules"') + expect(prompt).toContain("never invent ids") + expect(prompt).toContain("id=item-4") + expect(prompt).toContain("doc-1") + }) + + it("keeps illustrative examples separated from main instructions", () => { + const prompt = buildDistillPrompt("indicator", { + observations: [], + existingItems: [], + referencedDocumentIds: [], + }) + const main = prompt.slice(0, prompt.indexOf("## Illustrative examples")) + expect(main).not.toMatch(/gross margin|市盈率|\bPE\b|Fed|美联储|NVIDIA|英伟达/i) + expect(prompt).toContain("## Illustrative examples (finance vertical") + expect(prompt).toContain("(no pending observations)") + }) +}) diff --git a/src/domains/memory/distill-prompts.ts b/src/domains/memory/distill-prompts.ts new file mode 100644 index 00000000..3e9e70ca --- /dev/null +++ b/src/domains/memory/distill-prompts.ts @@ -0,0 +1,311 @@ +import type { + DistillObservationInput, + DistillPassKind, + ExistingMemoryContextItem, +} from "./distill-types" + +/** + * Distill prompts — three isolated passes over the same pending observation + * batch. Each pass sees ALL observations (no kind routing) and only the + * existing memories of kinds that pass may write. + */ + +const INDICATOR_OUTPUT_SCHEMA_BLOCK = `{ + "indicatorPrefs": [{ + "name": "string", + "aliases": ["string"], + "definition": "string", + "polarity": "higher_better|lower_better|context", + "importance": "core|secondary", + "formulaHint": "string (optional — omit if none)", + "abstractL0": "string", + "overviewL1": "string", + "confidence": 0.0, + "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } + }] +}` + +const EXPERIENCE_OUTPUT_SCHEMA_BLOCK = `{ + "stances": [{ + "statement": "string (the stance text; do not use a name field)", + "scope": "string", + "rationale": "string", + "abstractL0": "string", + "overviewL1": "string", + "confidence": 0.0, + "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } + }], + "decisionRules": [{ + "when": "string", + "then": "string", + "priority": "high|medium|low", + "rationale": "string", + "abstractL0": "string", + "overviewL1": "string", + "confidence": 0.0, + "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } + }] +}` + +const ENTITY_OUTPUT_SCHEMA_BLOCK = `{ + "entities": [{ + "name": "string", + "ticker": "string optional", + "aliases": ["string"], + "knowhereDocumentIds": ["only ids listed under REFERENCED DOCUMENT IDS"], + "reason": "string", + "abstractL0": "string", + "overviewL1": "string", + "confidence": 0.0, + "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } + }] +}` + +const INDICATOR_ILLUSTRATIVE_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) + +These show shape and judgement only. Distill whatever the observations support; +do not force the batch into this domain or these metric names. + +- Recurring evaluation metric named across clues → one indicatorPref (stable name + definition + polarity + importance). +- Same metric restated with a nuance → merge into the existing item, do not create a second. +- Skip: a one-off number question, document fact, or weak single-mention with no reusable criterion.` + +const EXPERIENCE_ILLUSTRATIVE_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) + +These show shape and judgement only. Distill whatever the observations support; +do not force the batch into this domain. + +- Durable judgement frame (e.g. long-horizon) → one stance (statement + scope + rationale). +- Reusable when → then discipline over the user's criteria → one decisionRule. +- Abstract away one-off instances; keep a single intent per rule. Split unrelated intents. +- Skip: process narration, document facts, or a preference that is only a metric definition (indicators are another pass).` + +const ENTITY_ILLUSTRATIVE_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) + +These show shape and judgement only. Distill whatever the observations support; +do not force the batch into this domain. + +- User actively tracks a named company/issuer across clues → one entity (name + reason; optional ticker/aliases). +- Same subject restated → merge; attach knowhereDocumentIds only from REFERENCED DOCUMENT IDS. +- Skip: a company mentioned only as a one-off fact question, or names that are not subjects of ongoing interest.` + +const INDICATOR_INSTRUCTIONS_BLOCK = `You DISTILL indicator preferences for a user's fluid memory. + +You receive a BATCH of raw observations (cheap clues about what the USER cares +about) plus existing indicator memories. Produce durable indicator_pref items +only. A separate pass handles stances, decision rules, and entities — never emit +those kinds here. + +Constraints: +- One stable topic/name per preference; merge overlapping or synonymous names. +- Capture "what the user repeatedly uses to evaluate", not one-off facts. +- Keep unrelated criteria as separate items; do not mix them into one payload. + +## What to emit + +indicatorPrefs — recurring metrics or criteria the user uses to evaluate things. +Fields: name, aliases, definition, polarity (higher_better | lower_better | context), +importance (core | secondary), optional formulaHint, abstractL0, overviewL1, +confidence, decision. + +## Language + +- Keep this instruction set and enum/field names in English. +- Write every free-text value in the same language the observations use. Do not + translate the user's terms into English unless the observations themselves used English. + +## Decision rules + +- Distill only from the observation batch evidence about the USER. +- Skip document facts, retrieved numbers, and weak/ephemeral clues. +- For every candidate, choose exactly one op against EXISTING MEMORIES: + - create — genuinely new indicator + - skip — already covered, or too weak + - merge — same indicator refined; emit the full merged fields and set targetItemId + - deprecate — user clearly reversed a stored indicator; set targetItemId +- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. +- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when explicitly stated. +- Omit optional fields instead of setting them to null. +- If nothing qualifies, return {"indicatorPrefs": []}.` + +const EXPERIENCE_INSTRUCTIONS_BLOCK = `You DISTILL stances and decision rules (insights) for a user's fluid memory. + +You receive a BATCH of raw observations plus existing stance/decision-rule +memories. Produce durable stances and decisionRules only. A separate pass +handles indicators and entities — never emit those kinds here. + +Constraints: +- Generalizable, reusable insight — not a process log of one session. +- Atomic scope: one intent per decisionRule; split if when would mix goals. +- Abstract away specific one-off entities/ids from the situation framing when the + rule itself is general; keep concrete names only when the insight requires them. +- Do not restate a bare metric definition as a decisionRule — that belongs to the indicator pass. + +## What to emit + +- stances — durable positions that shape how the user weighs evidence. + Fields: statement (required; do not invent a "name" field), scope, rationale, + abstractL0, overviewL1, confidence, decision. +- decisionRules — reusable when → then disciplines the user stated or clearly endorsed. + Fields: when, then, priority (high | medium | low), rationale, abstractL0, + overviewL1, confidence, decision. + +## Language + +- Keep this instruction set and enum/field names in English. +- Write every free-text value in the same language the observations use. Do not + translate the user's terms into English unless the observations themselves used English. + +## Decision rules + +- Distill only from the observation batch evidence about the USER. +- Skip document facts, small talk, and weak/ephemeral clues. +- For every candidate, choose exactly one op against EXISTING MEMORIES: + - create — genuinely new + - skip — already covered, or too weak + - merge — same insight refined; emit the full merged fields and set targetItemId + - deprecate — user clearly reversed a stored item; set targetItemId +- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. +- Prefer one record per insight. Do not invent a near-duplicate decisionRule for a stance that already encodes the same frame unless the user stated an explicit when → then action. +- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when explicitly stated. +- Omit optional fields instead of setting them to null. +- If nothing qualifies, return {"stances": [], "decisionRules": []}.` + +const ENTITY_INSTRUCTIONS_BLOCK = `You DISTILL entities of interest for a user's fluid memory. + +You receive a BATCH of raw observations plus existing entity memories. Produce +durable entity_of_interest items only. A separate pass handles indicators, +stances, and decision rules — never emit those kinds here. + +Constraints: +- Stable card for a subject the USER actively tracks. +- Merge overlapping names/aliases into one item; keep unrelated subjects separate. +- Attach document provenance only from ids listed under REFERENCED DOCUMENT IDS. + +## What to emit + +entities — named subjects the user is actively tracking. +Fields: name, optional ticker, aliases, reason (required), knowhereDocumentIds +(only from REFERENCED DOCUMENT IDS below; never invent ids), abstractL0, +overviewL1, confidence, decision. + +## Language + +- Keep this instruction set and enum/field names in English. +- Write every free-text value in the same language the observations use. Do not + translate the user's terms into English unless the observations themselves used English. + +## Decision rules + +- Distill only from the observation batch evidence about the USER. +- Skip one-off name drops, document facts, and weak/ephemeral mentions. +- For every candidate, choose exactly one op against EXISTING MEMORIES: + - create — genuinely new tracked subject + - skip — already covered, or too weak + - merge — same subject refined; emit the full merged fields and set targetItemId + - deprecate — user clearly stopped tracking / reversed; set targetItemId +- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. +- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when explicitly stated. +- Omit optional fields instead of setting them to null. +- If nothing qualifies, return {"entities": []}.` + +export type BuildDistillPromptInput = { + readonly observations: readonly DistillObservationInput[] + readonly existingItems: readonly ExistingMemoryContextItem[] + readonly referencedDocumentIds: readonly string[] +} + +export function buildDistillPrompt( + pass: DistillPassKind, + input: BuildDistillPromptInput, +): string { + switch (pass) { + case "indicator": + return assemblePrompt({ + instructions: INDICATOR_INSTRUCTIONS_BLOCK, + examples: INDICATOR_ILLUSTRATIVE_BLOCK, + outputSchema: INDICATOR_OUTPUT_SCHEMA_BLOCK, + input, + }) + case "experience": + return assemblePrompt({ + instructions: EXPERIENCE_INSTRUCTIONS_BLOCK, + examples: EXPERIENCE_ILLUSTRATIVE_BLOCK, + outputSchema: EXPERIENCE_OUTPUT_SCHEMA_BLOCK, + input, + }) + case "entity": + return assemblePrompt({ + instructions: ENTITY_INSTRUCTIONS_BLOCK, + examples: ENTITY_ILLUSTRATIVE_BLOCK, + outputSchema: ENTITY_OUTPUT_SCHEMA_BLOCK, + input, + }) + } +} + +function assemblePrompt(args: { + readonly instructions: string + readonly examples: string + readonly outputSchema: string + readonly input: BuildDistillPromptInput +}): string { + const existingBlock = + args.input.existingItems.length === 0 + ? "(no existing memories yet)" + : args.input.existingItems + .map( + (item) => + `- [${item.kind}] id=${item.id} :: ${item.abstractL0} :: ${item.payloadSummary}`, + ) + .join("\n") + + const documentsBlock = + args.input.referencedDocumentIds.length === 0 + ? "(no documents referenced in this batch)" + : args.input.referencedDocumentIds.join(", ") + + const observationsBlock = + args.input.observations.length === 0 + ? "(no pending observations)" + : args.input.observations + .map((observation) => formatObservation(observation)) + .join("\n\n") + + return `${args.instructions} + +${args.examples} + +## Output JSON schema (follow exactly; do not invent fields) + +${args.outputSchema} + +## EXISTING MEMORIES + +${existingBlock} + +## REFERENCED DOCUMENT IDS + +${documentsBlock} + +## PENDING OBSERVATIONS + +${observationsBlock}` +} + +function formatObservation(observation: DistillObservationInput): string { + const subject = + observation.subjectHint && observation.subjectHint.length > 0 + ? observation.subjectHint + : "(none)" + const docs = + observation.referencedDocumentIds.length === 0 + ? "(none)" + : observation.referencedDocumentIds.join(", ") + return `- id=${observation.id} + signal: ${observation.signal} + evidenceQuote: ${observation.evidenceQuote} + subjectHint: ${subject} + confidence: ${observation.confidence} + referencedDocumentIds: ${docs}` +} diff --git a/src/domains/memory/distill-trigger.test.ts b/src/domains/memory/distill-trigger.test.ts new file mode 100644 index 00000000..f4a09d10 --- /dev/null +++ b/src/domains/memory/distill-trigger.test.ts @@ -0,0 +1,84 @@ +import { afterEach, describe, expect, it, vi } from "vitest" + +import { DISTILL_COOLDOWN_MS, DISTILL_MIN_PENDING } from "./distill-config" + +const mocks = vi.hoisted(() => ({ + loggerError: vi.fn(), + loggerInfo: vi.fn(), + loggerWarn: vi.fn(), + trigger: vi.fn(), + countPendingObservations: vi.fn(), +})) + +vi.mock("@upstash/workflow", () => ({ + Client: class { + trigger = mocks.trigger + }, +})) + +vi.mock("@/lib/logger", () => ({ + logger: { + error: mocks.loggerError, + info: mocks.loggerInfo, + warn: mocks.loggerWarn, + }, +})) + +vi.mock("./service", () => ({ + memoryService: { + countPendingObservations: mocks.countPendingObservations, + }, +})) + +describe("triggerMemoryDistill", () => { + afterEach(async () => { + vi.clearAllMocks() + vi.useRealTimers() + delete process.env.QSTASH_TOKEN + delete process.env.NOTEBOOK_PUBLIC_URL + const { resetMemoryDistillTriggerStateForTests } = await import( + "./distill-trigger" + ) + resetMemoryDistillTriggerStateForTests() + vi.resetModules() + }) + + it("does not trigger when pending is below the plan threshold", async () => { + mocks.countPendingObservations.mockResolvedValue(DISTILL_MIN_PENDING - 1) + process.env.QSTASH_TOKEN = "qstash_token" + + const { triggerMemoryDistill } = await import("./distill-trigger") + await triggerMemoryDistill({ workspaceId: "workspace_1" }) + + expect(mocks.trigger).not.toHaveBeenCalled() + }) + + it("deduplicates workflow triggers only within a bounded cooldown", async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date("2026-06-30T00:00:00.000Z")) + process.env.QSTASH_TOKEN = "qstash_token" + process.env.NOTEBOOK_PUBLIC_URL = "https://notebook.example" + mocks.countPendingObservations.mockResolvedValue(DISTILL_MIN_PENDING) + mocks.trigger.mockResolvedValue({}) + + const { triggerMemoryDistill } = await import("./distill-trigger") + + await triggerMemoryDistill({ workspaceId: "workspace_1" }) + await triggerMemoryDistill({ workspaceId: "workspace_1" }) + + expect(mocks.trigger).toHaveBeenCalledTimes(1) + expect(mocks.trigger).toHaveBeenLastCalledWith({ + url: "https://notebook.example/api/memory/distill", + body: { workspaceId: "workspace_1" }, + workflowRunId: `workspace_1-${Math.floor( + new Date("2026-06-30T00:00:00.000Z").getTime() / DISTILL_COOLDOWN_MS, + )}`, + retries: 3, + }) + + vi.setSystemTime(new Date("2026-06-30T00:05:01.000Z")) + await triggerMemoryDistill({ workspaceId: "workspace_1" }) + + expect(mocks.trigger).toHaveBeenCalledTimes(2) + }) +}) diff --git a/src/domains/memory/distill-trigger.ts b/src/domains/memory/distill-trigger.ts new file mode 100644 index 00000000..a9bd0fd6 --- /dev/null +++ b/src/domains/memory/distill-trigger.ts @@ -0,0 +1,77 @@ +import "server-only" + +import { Client } from "@upstash/workflow" + +import { DISTILL_COOLDOWN_MS, DISTILL_MIN_PENDING } from "./distill-config" +import type { MemoryDistillPayload } from "./distill-workflow" +import { memoryService } from "./service" +import { logger } from "@/lib/logger" + +// Re-trigger protection: process-local cooldown + bucketed workflowRunId. +// Mirrors background-reconcile — same cooldown width keys both guards. + +const lastTriggeredAtByWorkspaceId: Map = new Map() + +function resolveBaseURL(): string { + return process.env.NOTEBOOK_PUBLIC_URL ?? "http://localhost:3000" +} + +/** + * Fire-and-forget distill trigger for one workspace. + * Caller should already know pending may be high; this re-checks the count, + * applies cooldown + bucketed workflowRunId, then enqueues QStash. + */ +export async function triggerMemoryDistill( + payload: MemoryDistillPayload, +): Promise { + const pendingCount = await memoryService.countPendingObservations( + payload.workspaceId, + ) + if (pendingCount < DISTILL_MIN_PENDING) return + + const now = Date.now() + const lastTriggeredAt = lastTriggeredAtByWorkspaceId.get(payload.workspaceId) + if ( + lastTriggeredAt !== undefined && + now - lastTriggeredAt < DISTILL_COOLDOWN_MS + ) { + return + } + lastTriggeredAtByWorkspaceId.set(payload.workspaceId, now) + + const token = process.env.QSTASH_TOKEN + if (!token) { + logger.warn("memory: skipping distill — QSTASH_TOKEN not set", { + workspaceId: payload.workspaceId, + pendingCount, + }) + lastTriggeredAtByWorkspaceId.delete(payload.workspaceId) + return + } + + const url = `${resolveBaseURL()}/api/memory/distill` + try { + await new Client({ token }).trigger({ + url, + body: payload, + workflowRunId: `${payload.workspaceId}-${Math.floor(now / DISTILL_COOLDOWN_MS)}`, + retries: 3, + }) + logger.info("memory: distill workflow triggered", { + workspaceId: payload.workspaceId, + pendingCount, + url, + }) + } catch (error) { + lastTriggeredAtByWorkspaceId.delete(payload.workspaceId) + logger.error("memory: failed to trigger distill workflow", { + workspaceId: payload.workspaceId, + message: error instanceof Error ? error.message : String(error), + }) + } +} + +/** Test helper: clear process-local cooldown map between cases. */ +export function resetMemoryDistillTriggerStateForTests(): void { + lastTriggeredAtByWorkspaceId.clear() +} diff --git a/src/domains/memory/distill-types.test.ts b/src/domains/memory/distill-types.test.ts new file mode 100644 index 00000000..b3b161d5 --- /dev/null +++ b/src/domains/memory/distill-types.test.ts @@ -0,0 +1,209 @@ +import { describe, expect, it } from "vitest" + +import { + entityDistillOutputSchema, + experienceDistillOutputSchema, + indicatorDistillOutputSchema, + memoryOperationsSchema, + summarizePayloadForContext, + toMemoryOperations, +} from "./distill-types" + +describe("distill output schemas", () => { + it("indicator pass accepts prefs and defaults empty array", () => { + expect(indicatorDistillOutputSchema.parse({})).toEqual({ + indicatorPrefs: [], + }) + const parsed = indicatorDistillOutputSchema.parse({ + indicatorPrefs: [ + { + name: "毛利率", + aliases: [], + definition: "毛利占营收", + polarity: "higher_better", + importance: "core", + abstractL0: "看重毛利率", + overviewL1: "用户用毛利率判断质量。", + confidence: 0.9, + decision: { op: "create" }, + }, + ], + }) + expect(parsed.indicatorPrefs).toHaveLength(1) + expect(parsed).not.toHaveProperty("stances") + }) + + it("experience pass accepts stance+rule when required fields are present", () => { + const parsed = experienceDistillOutputSchema.parse({ + stances: [ + { + statement: "长期持有", + scope: "投资 horizon", + rationale: "用户明确说长期", + abstractL0: "长期持有", + overviewL1: "用户以长期视角评估。", + confidence: 1, + decision: { op: "create" }, + }, + ], + decisionRules: [ + { + when: "毛利率连续两季下滑", + then: "减仓观望", + priority: "high", + rationale: "用户自述纪律", + abstractL0: "毛利率下滑则减仓", + overviewL1: "连续两季下滑时减仓观望。", + confidence: 0.95, + decision: { op: "create" }, + }, + ], + }) + expect(parsed.stances[0]?.statement).toBe("长期持有") + expect(parsed.decisionRules).toHaveLength(1) + expect(parsed).not.toHaveProperty("indicatorPrefs") + }) + + it("rejects stance that uses name instead of statement (no coerce)", () => { + expect(() => + experienceDistillOutputSchema.parse({ + stances: [ + { + name: "长期持有", + scope: "投资", + rationale: "用户明确说长期", + abstractL0: "长期持有", + overviewL1: "用户以长期视角评估。", + confidence: 1, + decision: { op: "create" }, + }, + ], + }), + ).toThrow() + }) + + it("rejects entity missing reason (no fill from abstractL0)", () => { + expect(() => + entityDistillOutputSchema.parse({ + entities: [ + { + name: "英伟达", + aliases: ["NVIDIA"], + knowhereDocumentIds: ["doc-1"], + abstractL0: "持续跟踪英伟达", + overviewL1: "用户把英伟达列为跟踪标的。", + confidence: 0.8, + decision: { op: "merge", targetItemId: "item-4" }, + }, + ], + }), + ).toThrow() + }) + + it("entity pass accepts when reason is present", () => { + const parsed = entityDistillOutputSchema.parse({ + entities: [ + { + name: "英伟达", + aliases: ["NVIDIA"], + knowhereDocumentIds: ["doc-1"], + reason: "用户持续跟踪", + abstractL0: "持续跟踪英伟达", + overviewL1: "用户把英伟达列为跟踪标的。", + confidence: 0.8, + decision: { op: "merge", targetItemId: "item-4" }, + }, + ], + }) + expect(parsed.entities[0]?.reason).toBe("用户持续跟踪") + }) + + it("toMemoryOperations expands each pass into the full four-array record", () => { + expect( + toMemoryOperations("indicator", { + indicatorPrefs: [], + }), + ).toEqual({ + indicatorPrefs: [], + stances: [], + decisionRules: [], + entities: [], + }) + + const experience = toMemoryOperations( + "experience", + experienceDistillOutputSchema.parse({ + stances: [ + { + statement: "长期", + scope: "投资", + rationale: "用户说的", + abstractL0: "长期", + overviewL1: "长期视角。", + confidence: 1, + decision: { op: "create" }, + }, + ], + }), + ) + expect(experience.stances).toHaveLength(1) + expect(experience.indicatorPrefs).toEqual([]) + expect(experience.entities).toEqual([]) + + const entity = toMemoryOperations( + "entity", + entityDistillOutputSchema.parse({ + entities: [ + { + name: "英伟达", + aliases: [], + knowhereDocumentIds: [], + reason: "跟踪", + abstractL0: "跟踪英伟达", + overviewL1: "用户跟踪英伟达。", + confidence: 0.7, + decision: { op: "create" }, + }, + ], + }), + ) + expect(entity.entities).toHaveLength(1) + expect(entity.decisionRules).toEqual([]) + }) + + it("memoryOperationsSchema parses the full four-array shape", () => { + const parsed = memoryOperationsSchema.parse({}) + expect(parsed).toEqual({ + indicatorPrefs: [], + stances: [], + decisionRules: [], + entities: [], + }) + }) +}) + +describe("summarizePayloadForContext", () => { + it("summarizes each kind for existing-memory context lines", () => { + expect( + summarizePayloadForContext("indicator_pref", { + name: "毛利率", + definition: "毛利占营收", + }), + ).toBe("毛利率 — 毛利占营收") + expect( + summarizePayloadForContext("stance", { statement: "长期持有" }), + ).toBe("长期持有") + expect( + summarizePayloadForContext("decision_rule", { + when: "下滑", + then: "减仓", + }), + ).toBe("下滑 => 减仓") + expect( + summarizePayloadForContext("entity_of_interest", { + name: "英伟达", + ticker: "NVDA", + }), + ).toBe("英伟达 NVDA") + }) +}) diff --git a/src/domains/memory/distill-types.ts b/src/domains/memory/distill-types.ts new file mode 100644 index 00000000..44b22749 --- /dev/null +++ b/src/domains/memory/distill-types.ts @@ -0,0 +1,190 @@ +import { z } from "zod" + +import { + decisionRulePayloadSchema, + entityOfInterestPayloadSchema, + indicatorPreferencePayloadSchema, + stancePayloadSchema, + type FluidMemoryKind, +} from "./types" + +/** + * Distill-pass LLM contracts. + * + * Three separate structured-output schemas (indicator / experience / + * entity). Capture never emits these shapes — distill is the only writer + * of create/merge/deprecate decisions over `fluid_memory_items`. + */ + +function nullToUndefined(value: unknown): unknown { + return value === null ? undefined : value +} + +const decisionSchema = z.object({ + op: z.enum(["create", "skip", "merge", "deprecate"]), + targetItemId: z.preprocess( + nullToUndefined, + z + .string() + .optional() + .describe( + "Required for merge/deprecate: id of the existing memory item. Omit for create/skip.", + ), + ), + reason: z.preprocess( + nullToUndefined, + z + .string() + .optional() + .describe("Short justification, especially for skip/merge/deprecate."), + ), +}) + +const memorySidecarFields = { + abstractL0: z + .string() + .min(1) + .describe("One line, <= 30 words: the essence of this insight."), + overviewL1: z + .string() + .min(1) + .describe("2-3 sentences: what it means and when it applies."), + confidence: z + .number() + .min(0) + .max(1) + .describe("How explicitly the user stated this across the batch (1 = explicit)."), + decision: decisionSchema, +} + +const stanceEntrySchema = stancePayloadSchema.extend(memorySidecarFields) + +const entityEntrySchema = + entityOfInterestPayloadSchema.extend(memorySidecarFields) + +const indicatorEntrySchema = + indicatorPreferencePayloadSchema.extend(memorySidecarFields) + +const decisionRuleEntrySchema = + decisionRulePayloadSchema.extend(memorySidecarFields) + +/** Full four-array shape consumed by resolveMemoryOperations. */ +export const memoryOperationsSchema = z.object({ + indicatorPrefs: z.array(indicatorEntrySchema).default([]), + stances: z.array(stanceEntrySchema).default([]), + decisionRules: z.array(decisionRuleEntrySchema).default([]), + entities: z.array(entityEntrySchema).default([]), +}) + +export type MemoryOperations = z.infer + +/** Pass 1 — indicator preferences only. */ +export const indicatorDistillOutputSchema = z.object({ + indicatorPrefs: z.array(indicatorEntrySchema).default([]), +}) + +/** Pass 2 — stances + decision rules. */ +export const experienceDistillOutputSchema = z.object({ + stances: z.array(stanceEntrySchema).default([]), + decisionRules: z.array(decisionRuleEntrySchema).default([]), +}) + +/** Pass 3 — entities of interest only. */ +export const entityDistillOutputSchema = z.object({ + entities: z.array(entityEntrySchema).default([]), +}) + +export type IndicatorDistillOutput = z.infer +export type ExperienceDistillOutput = z.infer< + typeof experienceDistillOutputSchema +> +export type EntityDistillOutput = z.infer + +export const distillPassKinds = [ + "indicator", + "experience", + "entity", +] as const + +export type DistillPassKind = (typeof distillPassKinds)[number] + +/** Pending observation row shape fed into distill prompts (batch evidence). */ +export type DistillObservationInput = { + readonly id: string + readonly signal: string + readonly evidenceQuote: string + readonly subjectHint: string | null + readonly confidence: number + readonly referencedDocumentIds: readonly string[] +} + +export type ExistingMemoryContextItem = { + readonly id: string + readonly kind: FluidMemoryKind + readonly abstractL0: string + readonly payloadSummary: string +} + +/** Expand a single-pass LLM object into the full MemoryOperations record. */ +export function toMemoryOperations( + pass: DistillPassKind, + output: + | IndicatorDistillOutput + | ExperienceDistillOutput + | EntityDistillOutput, +): MemoryOperations { + switch (pass) { + case "indicator": { + const typed = output as IndicatorDistillOutput + return { + indicatorPrefs: typed.indicatorPrefs, + stances: [], + decisionRules: [], + entities: [], + } + } + case "experience": { + const typed = output as ExperienceDistillOutput + return { + indicatorPrefs: [], + stances: typed.stances, + decisionRules: typed.decisionRules, + entities: [], + } + } + case "entity": { + const typed = output as EntityDistillOutput + return { + indicatorPrefs: [], + stances: [], + decisionRules: [], + entities: typed.entities, + } + } + } +} + +/** Compact payload label for existing-item context in distill prompts. */ +export function summarizePayloadForContext( + kind: FluidMemoryKind, + payload: unknown, +): string { + if (!payload || typeof payload !== "object") return "" + const record = payload as Record + switch (kind) { + case "indicator_pref": + return [record.name, record.definition] + .filter((part) => typeof part === "string" && part.length > 0) + .join(" — ") + case "stance": + return typeof record.statement === "string" ? record.statement : "" + case "decision_rule": + return [record.when, record.then] + .filter((part) => typeof part === "string" && part.length > 0) + .join(" => ") + case "entity_of_interest": + return [record.name, record.ticker] + .filter((part) => typeof part === "string" && part.length > 0) + .join(" ") + } +} diff --git a/src/domains/memory/distill-workflow.test.ts b/src/domains/memory/distill-workflow.test.ts new file mode 100644 index 00000000..a96b576a --- /dev/null +++ b/src/domains/memory/distill-workflow.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest" + +import { + normalizeMemoryDistillPayload, + toDistillObservationInput, +} from "./distill-workflow" +import type { FluidObservation } from "@/infrastructure/db/schema" + +describe("normalizeMemoryDistillPayload", () => { + it("accepts a workspace id", () => { + expect(normalizeMemoryDistillPayload({ workspaceId: "ws-1" })).toEqual({ + workspaceId: "ws-1", + }) + }) + + it("rejects missing or blank workspace id", () => { + expect(normalizeMemoryDistillPayload(null)).toBeNull() + expect(normalizeMemoryDistillPayload({})).toBeNull() + expect(normalizeMemoryDistillPayload({ workspaceId: " " })).toBeNull() + }) +}) + +describe("toDistillObservationInput", () => { + it("maps a pending row without inventing fields", () => { + const row = { + id: "obs-1", + workspaceId: "ws-1", + sourceMessageId: "msg-1", + signal: "看重毛利率", + evidenceQuote: "毛利率是核心", + subjectHint: "毛利率", + referencedDocumentIds: ["doc-1", ""], + confidence: 0.9, + status: "pending", + createdAt: new Date("2026-06-30T00:00:00.000Z"), + consumedAt: null, + } as FluidObservation + + expect(toDistillObservationInput(row)).toEqual({ + id: "obs-1", + signal: "看重毛利率", + evidenceQuote: "毛利率是核心", + subjectHint: "毛利率", + confidence: 0.9, + referencedDocumentIds: ["doc-1"], + }) + }) +}) diff --git a/src/domains/memory/distill-workflow.ts b/src/domains/memory/distill-workflow.ts new file mode 100644 index 00000000..c491a718 --- /dev/null +++ b/src/domains/memory/distill-workflow.ts @@ -0,0 +1,231 @@ +import "server-only" + +import type { WorkflowContext } from "@upstash/workflow" + +import { + DISTILL_BATCH_MAX, + DISTILL_CONSUMED_RETENTION_MS, + DISTILL_DEDUP_CANDIDATES_PER_KIND, +} from "./distill-config" +import { distillMemoryPass } from "./distill-model" +import { + summarizePayloadForContext, + type DistillObservationInput, + type DistillPassKind, + type ExistingMemoryContextItem, + type MemoryOperations, +} from "./distill-types" +import { resolveMemoryOperations } from "./resolve-operations" +import { tokenizeMemoryText } from "./search-index" +import { memoryService } from "./service" +import type { FluidMemoryKind } from "./types" +import { isFluidMemoryKind } from "./types" +import type { FluidMemoryItem, FluidObservation } from "@/infrastructure/db/schema" +import { logger } from "@/lib/logger" + +export type MemoryDistillPayload = { + readonly workspaceId: string +} + +type MemoryDistillWorkflowContext = Pick< + WorkflowContext, + "run" +> + +const PASS_KINDS: readonly { + readonly pass: DistillPassKind + readonly kinds: readonly FluidMemoryKind[] +}[] = [ + { pass: "indicator", kinds: ["indicator_pref"] }, + { pass: "experience", kinds: ["stance", "decision_rule"] }, + { pass: "entity", kinds: ["entity_of_interest"] }, +] + +export function normalizeMemoryDistillPayload( + raw: unknown, +): MemoryDistillPayload | null { + if (!raw || typeof raw !== "object") return null + const workspaceId = getNonEmptyString( + (raw as Record).workspaceId, + ) + if (!workspaceId) return null + return { workspaceId } +} + +/** + * Periodic distill: pending observations → three typed passes → resolve → + * write fluid_memory_items and mark the batch consumed. Capture never writes + * the permanent layer; this job is the only writer. + */ +export async function runMemoryDistillWorkflow(input: { + readonly context: MemoryDistillWorkflowContext + readonly payload: MemoryDistillPayload +}): Promise { + const { context, payload } = input + + const batch = await context.run("select-batch", () => + memoryService.listPendingObservations( + payload.workspaceId, + DISTILL_BATCH_MAX, + ), + ) + if (batch.length === 0) { + logger.info("memory: distill skipped — no pending observations", { + workspaceId: payload.workspaceId, + }) + return + } + + const observationInputs = batch.map(toDistillObservationInput) + const referencedDocumentIds = unionDocumentIds(batch) + const queryTokens = tokenizeBatch(observationInputs) + + const candidatesByKind = await context.run("load-candidates", async () => { + const result: Partial> = {} + for (const kind of [ + "indicator_pref", + "stance", + "decision_rule", + "entity_of_interest", + ] as const) { + result[kind] = await memoryService.findDedupCandidates( + payload.workspaceId, + kind, + queryTokens, + DISTILL_DEDUP_CANDIDATES_PER_KIND, + ) + } + return result + }) + + const passOperations: MemoryOperations[] = [] + for (const { pass, kinds } of PASS_KINDS) { + const existingItems = kinds.flatMap((kind) => + (candidatesByKind[kind] ?? []).flatMap((item) => { + const mapped = toExistingMemoryContextItem(item) + return mapped ? [mapped] : [] + }), + ) + const operations = await context.run(`distill-${pass}`, () => + distillMemoryPass({ + pass, + workspaceId: payload.workspaceId, + observations: observationInputs, + existingItems, + referencedDocumentIds, + }), + ) + // Null = model failure for this pass only; other passes still apply. + if (operations) passOperations.push(operations) + } + + if (passOperations.length === 0) { + logger.warn( + "memory: distill aborted — all passes failed; batch left pending", + { + workspaceId: payload.workspaceId, + batchSize: batch.length, + }, + ) + return + } + + const existingItemRefs = Object.values(candidatesByKind) + .flat() + .map((item) => ({ + id: item.id, + kind: item.kind, + status: item.status, + payload: item.payload, + })) + + const resolved = passOperations.flatMap((operations) => + resolveMemoryOperations({ + operations, + existingItems: existingItemRefs, + referencedDocumentIds, + }), + ) + + const applied = await context.run("apply-and-consume", () => + memoryService.applyDistillBatch({ + workspaceId: payload.workspaceId, + sourceMessageId: null, + operations: resolved, + observationIds: batch.map((row) => row.id), + }), + ) + + const deleted = await context.run("retention", () => + memoryService.deleteExpiredConsumedObservations( + new Date(Date.now() - DISTILL_CONSUMED_RETENTION_MS), + ), + ) + + logger.info("memory: distill workflow finished", { + workspaceId: payload.workspaceId, + batchSize: batch.length, + resolvedCount: resolved.length, + diffCount: applied.diffs.length, + consumedCount: applied.consumedCount, + retentionDeleted: deleted, + }) +} + +export function toDistillObservationInput( + row: FluidObservation, +): DistillObservationInput { + const documentIds = Array.isArray(row.referencedDocumentIds) + ? row.referencedDocumentIds.filter( + (id): id is string => typeof id === "string" && id.length > 0, + ) + : [] + return { + id: row.id, + signal: row.signal, + evidenceQuote: row.evidenceQuote, + subjectHint: row.subjectHint, + confidence: row.confidence, + referencedDocumentIds: documentIds, + } +} + +function toExistingMemoryContextItem( + item: FluidMemoryItem, +): ExistingMemoryContextItem | null { + if (!isFluidMemoryKind(item.kind)) return null + return { + id: item.id, + kind: item.kind, + abstractL0: item.abstractL0, + payloadSummary: summarizePayloadForContext(item.kind, item.payload), + } +} + +function tokenizeBatch( + observations: readonly DistillObservationInput[], +): string[] { + const text = observations + .map((observation) => + [observation.signal, observation.subjectHint ?? "", observation.evidenceQuote] + .filter((part) => part.length > 0) + .join(" "), + ) + .join(" ") + return tokenizeMemoryText(text).map((entry) => entry.token) +} + +function unionDocumentIds(rows: readonly FluidObservation[]): string[] { + const ids = new Set() + for (const row of rows) { + if (!Array.isArray(row.referencedDocumentIds)) continue + for (const id of row.referencedDocumentIds) { + if (typeof id === "string" && id.length > 0) ids.add(id) + } + } + return [...ids] +} + +function getNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null +} diff --git a/src/domains/memory/extract-workflow.test.ts b/src/domains/memory/extract-workflow.test.ts new file mode 100644 index 00000000..41205a55 --- /dev/null +++ b/src/domains/memory/extract-workflow.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest" + +import { normalizeMemoryExtractPayload } from "./extract-workflow" + +describe("normalizeMemoryExtractPayload", () => { + it("accepts a complete payload", () => { + expect( + normalizeMemoryExtractPayload({ + workspaceId: "ws-1", + threadId: "th-1", + userMessageId: "u-1", + assistantMessageId: "a-1", + }), + ).toEqual({ + workspaceId: "ws-1", + threadId: "th-1", + userMessageId: "u-1", + assistantMessageId: "a-1", + }) + }) + + it("rejects missing or blank fields", () => { + expect(normalizeMemoryExtractPayload(null)).toBeNull() + expect( + normalizeMemoryExtractPayload({ + workspaceId: "ws-1", + threadId: "th-1", + userMessageId: "u-1", + }), + ).toBeNull() + expect( + normalizeMemoryExtractPayload({ + workspaceId: " ", + threadId: "th-1", + userMessageId: "u-1", + assistantMessageId: "a-1", + }), + ).toBeNull() + }) +}) diff --git a/src/domains/memory/extract-workflow.ts b/src/domains/memory/extract-workflow.ts index aa6b5ee3..ec0a538b 100644 --- a/src/domains/memory/extract-workflow.ts +++ b/src/domains/memory/extract-workflow.ts @@ -2,15 +2,9 @@ import "server-only" import type { WorkflowContext } from "@upstash/workflow" -import { extractMemoryOperations } from "./extraction-model" -import { - summarizePayloadForContext, - type ExistingMemoryContextItem, -} from "./prompts" -import { resolveMemoryOperations } from "./resolve-operations" -import { tokenizeMemoryText } from "./search-index" +import { triggerMemoryDistill } from "./distill-trigger" +import { captureObservations } from "./extraction-model" import { memoryService } from "./service" -import { fluidMemoryKinds, isFluidMemoryKind } from "./types" import { chatThreadService } from "@/domains/chat/thread-service" import { logger } from "@/lib/logger" @@ -26,15 +20,6 @@ type MemoryExtractWorkflowContext = Pick< "run" > -/** Prompt-context item that also carries status/payload for resolution. */ -type MemoryWorkflowItem = ExistingMemoryContextItem & { - readonly status: string - readonly payload: unknown -} - -/** Per-kind cap on lexical neighbors fed into the merge-decision prompt. */ -const DEDUP_CANDIDATES_PER_KIND = 8 - export function normalizeMemoryExtractPayload( raw: unknown, ): MemoryExtractPayload | null { @@ -50,6 +35,11 @@ export function normalizeMemoryExtractPayload( return { workspaceId, threadId, userMessageId, assistantMessageId } } +/** + * Per-turn coarse capture: load the turn → LLM observations → append-only + * insert into fluid_observations. Never writes fluid_memory_items (distill + * owns that). After persist, maybe-trigger distill when pending is high enough. + */ export async function runMemoryExtractWorkflow(input: { readonly context: MemoryExtractWorkflowContext readonly payload: MemoryExtractPayload @@ -77,73 +67,42 @@ export async function runMemoryExtractWorkflow(input: { } }) if (!turn) { - logger.warn("memory: extract skipped — turn messages not found", { + logger.warn("memory: capture skipped — turn messages not found", { workspaceId: payload.workspaceId, threadId: payload.threadId, }) return } - const existingItems = await context.run("retrieve-candidates", async () => { - const queryTokens = tokenizeMemoryText(turn.userText).map( - (entry) => entry.token, - ) - if (queryTokens.length === 0) return [] - - const byId = new Map() - for (const kind of fluidMemoryKinds) { - const items = await memoryService.findDedupCandidates( - payload.workspaceId, - kind, - queryTokens, - DEDUP_CANDIDATES_PER_KIND, - ) - for (const item of items) { - if (!isFluidMemoryKind(item.kind) || byId.has(item.id)) continue - byId.set(item.id, { - id: item.id, - kind: item.kind, - status: item.status, - payload: item.payload, - abstractL0: item.abstractL0, - payloadSummary: summarizePayloadForContext(item.kind, item.payload), - }) - } - } - return [...byId.values()] - }) - - const operations = await context.run("extract-operations", () => - extractMemoryOperations({ + const observations = await context.run("capture", () => + captureObservations({ workspaceId: payload.workspaceId, userText: turn.userText, assistantText: turn.assistantText, referencedDocumentIds: turn.referencedDocumentIds, - existingItems, }), ) - if (!operations) return + if (!observations) return - const applied = await context.run("apply-operations", async () => { - const resolved = resolveMemoryOperations({ - operations, - existingItems, + const inserted = await context.run("persist-observations", async () => { + if (observations.length === 0) return [] + return memoryService.insertObservations({ + workspaceId: payload.workspaceId, + sourceMessageId: payload.assistantMessageId, referencedDocumentIds: turn.referencedDocumentIds, + observations, }) - if (resolved.length === 0) return null - return memoryService.applyOperations( - payload.workspaceId, - payload.assistantMessageId, - resolved, - ) }) - logger.info("memory: extract workflow finished", { + await context.run("maybe-trigger-distill", async () => { + await triggerMemoryDistill({ workspaceId: payload.workspaceId }) + }) + + logger.info("memory: capture workflow finished", { workspaceId: payload.workspaceId, threadId: payload.threadId, assistantMessageId: payload.assistantMessageId, - candidateCount: existingItems.length, - appliedOperations: applied?.map((operation) => operation.op) ?? [], + observationCount: inserted.length, }) } diff --git a/src/domains/memory/extraction-model.ts b/src/domains/memory/extraction-model.ts index fa490115..e013e36f 100644 --- a/src/domains/memory/extraction-model.ts +++ b/src/domains/memory/extraction-model.ts @@ -3,11 +3,11 @@ import "server-only" import { generateObject } from "ai" import { - buildMemoryExtractionPrompt, - memoryOperationsSchema, - type ExistingMemoryContextItem, - type MemoryOperations, -} from "./prompts" + captureOutputSchema, + type CaptureOutput, + type CapturedObservation, +} from "./observation-types" +import { buildCapturePrompt } from "./prompts" import { CHAT_MODEL } from "@/lib/ai" import { summarizeUnknownError } from "@/lib/format-log-value" import { logger } from "@/lib/logger" @@ -15,35 +15,34 @@ import { logger } from "@/lib/logger" const MEMORY_EXTRACTION_MODEL = process.env.MEMORY_EXTRACTION_MODEL ?? CHAT_MODEL /** - * One structured-output call: turn + existing active memories in, typed - * operations out. Best-effort by design — this runs as a background job, - * so a model failure skips the turn (logged) instead of degrading through - * fallbacks; the insight typically resurfaces in a later turn. + * One structured-output call: conversation turn in, raw observations out. + * Best-effort — this runs as a background job, so a model failure skips the + * turn (logged) instead of degrading through fallbacks; the clue typically + * resurfaces in a later turn. */ -export async function extractMemoryOperations(input: { +export async function captureObservations(input: { readonly workspaceId: string readonly userText: string readonly assistantText: string readonly referencedDocumentIds: readonly string[] - readonly existingItems: readonly ExistingMemoryContextItem[] -}): Promise { +}): Promise { try { const response = await generateObject({ model: MEMORY_EXTRACTION_MODEL, - schema: memoryOperationsSchema, + schema: captureOutputSchema, messages: [ { role: "user", - content: buildMemoryExtractionPrompt(input), + content: buildCapturePrompt(input), }, ], }) - return response.object + const output: CaptureOutput = response.object + return output.observations } catch (error) { - logger.warn("memory: extraction model call failed; skipping turn", { + logger.warn("memory: capture model call failed; skipping turn", { workspaceId: input.workspaceId, model: MEMORY_EXTRACTION_MODEL, - existingItemCount: input.existingItems.length, error: summarizeUnknownError(error), }) return null diff --git a/src/domains/memory/observation-types.test.ts b/src/domains/memory/observation-types.test.ts new file mode 100644 index 00000000..0a99eb6c --- /dev/null +++ b/src/domains/memory/observation-types.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest" + +import { + captureOutputSchema, + capturedObservationSchema, +} from "./observation-types" + +describe("capturedObservationSchema", () => { + it("accepts a full observation", () => { + const parsed = capturedObservationSchema.parse({ + signal: "看重毛利率", + evidenceQuote: "毛利率是核心", + subjectHint: "毛利率", + confidence: 0.9, + }) + expect(parsed.subjectHint).toBe("毛利率") + expect(parsed).not.toHaveProperty("kindHint") + }) + + it("coerces null subjectHint to undefined (single preprocess)", () => { + const parsed = capturedObservationSchema.parse({ + signal: "长期持有", + evidenceQuote: "我做长期投资", + subjectHint: null, + confidence: 1, + }) + expect(parsed.subjectHint).toBeUndefined() + }) + + it("strips unknown kindHint if the model still emits it", () => { + const parsed = capturedObservationSchema.parse({ + signal: "看重毛利率", + evidenceQuote: "毛利率是核心", + kindHint: "indicator", + confidence: 0.9, + }) + expect(parsed).not.toHaveProperty("kindHint") + }) + + it("rejects empty signal or evidenceQuote", () => { + expect(() => + capturedObservationSchema.parse({ + signal: "", + evidenceQuote: "x", + confidence: 0.5, + }), + ).toThrow() + expect(() => + capturedObservationSchema.parse({ + signal: "x", + evidenceQuote: "", + confidence: 0.5, + }), + ).toThrow() + }) +}) + +describe("captureOutputSchema", () => { + it("defaults missing observations to empty array", () => { + expect(captureOutputSchema.parse({})).toEqual({ observations: [] }) + }) + + it("parses a batch of observations", () => { + const parsed = captureOutputSchema.parse({ + observations: [ + { + signal: "跟踪英伟达", + evidenceQuote: "英伟达一直在跟踪", + subjectHint: "英伟达", + confidence: 0.8, + }, + ], + }) + expect(parsed.observations).toHaveLength(1) + expect(parsed.observations[0]?.subjectHint).toBe("英伟达") + expect(parsed.observations[0]).not.toHaveProperty("kindHint") + }) +}) diff --git a/src/domains/memory/observation-types.ts b/src/domains/memory/observation-types.ts new file mode 100644 index 00000000..63f75987 --- /dev/null +++ b/src/domains/memory/observation-types.ts @@ -0,0 +1,50 @@ +import { z } from "zod" + +/** + * Coarse-capture contract for the raw observation layer. + * + * Capture records durable USER clues (what the user cares about) only. + * It does not classify into final memory kinds, does not dedup, and never + * writes `fluid_memory_items`. `subjectHint` is an optional topic anchor + * for later clustering — distill owns authoritative typing and merge. + */ + +/** Single concentrated null→undefined coerce for optional capture fields. */ +function nullToUndefined(value: unknown): unknown { + return value === null ? undefined : value +} + +export const capturedObservationSchema = z.object({ + signal: z + .string() + .min(1) + .describe( + "One durable clue about what the USER cares about, in the user's language.", + ), + evidenceQuote: z + .string() + .min(1) + .describe("Short verbatim snippet from the USER turn that supports signal."), + subjectHint: z.preprocess( + nullToUndefined, + z + .string() + .min(1) + .optional() + .describe( + "Optional short topic anchor (metric name, company, topic). Prefer omit when none.", + ), + ), + confidence: z + .number() + .min(0) + .max(1) + .describe("How explicitly the user stated this (1 = explicit)."), +}) + +export const captureOutputSchema = z.object({ + observations: z.array(capturedObservationSchema).default([]), +}) + +export type CapturedObservation = z.infer +export type CaptureOutput = z.infer diff --git a/src/domains/memory/prompts.test.ts b/src/domains/memory/prompts.test.ts index 2ae15ff2..2f7c83f7 100644 --- a/src/domains/memory/prompts.test.ts +++ b/src/domains/memory/prompts.test.ts @@ -1,20 +1,38 @@ import { describe, expect, it } from "vitest" -import { buildMemoryExtractionPrompt } from "./prompts" +import { buildCapturePrompt } from "./prompts" -describe("buildMemoryExtractionPrompt", () => { - const prompt = buildMemoryExtractionPrompt({ +describe("buildCapturePrompt", () => { + const prompt = buildCapturePrompt({ userText: "毛利率是核心。", assistantText: "明白。", referencedDocumentIds: ["doc-1"], - existingItems: [], }) - it("keeps main instructions domain-agnostic", () => { + it("keeps main instructions domain-agnostic and capture-only", () => { const main = prompt.slice(0, prompt.indexOf("## Illustrative examples")) expect(main).not.toMatch(/gross margin|市盈率|\bPE\b|Fed|美联储|NVIDIA|英伟达/i) + expect(main).toContain("RAW OBSERVATIONS") + expect(main).toContain("Do NOT classify observations into those kinds") + expect(main).toContain("Do NOT emit any") + expect(main).toContain("kind / type / category field") + expect(main).toContain("Do NOT invent") + expect(main).toContain("create/merge/deprecate operations") + expect(main).not.toContain("kindHint") + expect(main).not.toContain("EXISTING MEMORIES") + expect(main).not.toContain("indicatorPrefs") + expect(main).not.toContain("decisionRules") expect(main).toContain("Write every free-text value") - expect(main).toMatch(/same language the\s+USER wrote/) + expect(main).toMatch(/same\s+language the USER wrote/) + }) + + it("output schema has no early kind classification field", () => { + const schema = prompt.slice(prompt.indexOf("## Output JSON schema")) + expect(schema).not.toContain("kindHint") + expect(schema).toContain("subjectHint") + expect(schema).toContain("signal") + expect(schema).toContain("evidenceQuote") + expect(schema).toContain("confidence") }) it("keeps illustrative examples in a separate section", () => { @@ -28,8 +46,9 @@ describe("buildMemoryExtractionPrompt", () => { expect(examples).toContain("do not force the conversation into this domain") }) - it("still injects turn context after the fixed blocks", () => { + it("injects turn context and referenced docs; no existing-memory block", () => { expect(prompt).toContain("[user]\n毛利率是核心。") expect(prompt).toContain("doc-1") + expect(prompt).not.toContain("## EXISTING MEMORIES") }) }) diff --git a/src/domains/memory/prompts.ts b/src/domains/memory/prompts.ts index e2fbeb0c..2f12965f 100644 --- a/src/domains/memory/prompts.ts +++ b/src/domains/memory/prompts.ts @@ -1,235 +1,80 @@ -import { z } from "zod" - -import { - decisionRulePayloadSchema, - entityOfInterestPayloadSchema, - indicatorPreferencePayloadSchema, - stancePayloadSchema, - type FluidMemoryKind, -} from "./types" - /** - * LLM contract for fluid-memory extraction. + * Coarse-capture prompt for the raw observation layer. * - * Design borrowed from OpenViking's session-commit extraction (schema-driven - * typed operations, prefetch-then-decide), reimplemented as a single - * structured-output call: the model sees the turn plus lexically retrieved - * dedup candidates and directly outputs per-kind operations - * (create / skip / merge / deprecate), mirroring OpenViking's generated - * operations model without its ReAct tool loop. + * Capture extracts durable USER points of concern only. It does not classify + * into final memory kinds, does not dedup against existing items, and does not + * emit create/merge/deprecate decisions — those belong to distill. */ -const decisionSchema = z.object({ - op: z.enum(["create", "skip", "merge", "deprecate"]), - targetItemId: z.preprocess( - (value) => (value === null ? undefined : value), - z - .string() - .optional() - .describe( - "Required for merge/deprecate: the id of the existing memory item this operation targets. Omit for create/skip.", - ), - ), - reason: z.preprocess( - (value) => (value === null ? undefined : value), - z - .string() - .optional() - .describe("Short justification, especially for skip/merge/deprecate."), - ), -}) - -const memorySidecarFields = { - abstractL0: z - .string() - .min(1) - .describe("One line, <= 30 words: the essence of this insight."), - overviewL1: z - .string() - .min(1) - .describe("2-3 sentences: what it means and when it applies."), - confidence: z - .number() - .min(0) - .max(1) - .describe("How explicitly the user stated this (1 = explicit)."), - decision: decisionSchema, -} - -const stanceEntrySchema = z.preprocess((value) => { - if (!value || typeof value !== "object" || Array.isArray(value)) return value - const record = value as Record - // Models sometimes emit "name" for a stance; the contract field is "statement". - if ( - (typeof record.statement !== "string" || record.statement.length === 0) && - typeof record.name === "string" && - record.name.length > 0 - ) { - const { name, ...rest } = record - return { ...rest, statement: name } - } - return value -}, stancePayloadSchema.extend(memorySidecarFields)) - -const entityEntrySchema = z.preprocess((value) => { - if (!value || typeof value !== "object" || Array.isArray(value)) return value - const record = value as Record - // Keep provenance reason required; if the model omitted it, fall back to L0. - if ( - (typeof record.reason !== "string" || record.reason.length === 0) && - typeof record.abstractL0 === "string" && - record.abstractL0.length > 0 - ) { - return { ...record, reason: record.abstractL0 } - } - return value -}, entityOfInterestPayloadSchema.extend(memorySidecarFields)) - -export const memoryOperationsSchema = z.object({ - indicatorPrefs: z - .array(indicatorPreferencePayloadSchema.extend(memorySidecarFields)) - .default([]), - stances: z.array(stanceEntrySchema).default([]), - decisionRules: z - .array(decisionRulePayloadSchema.extend(memorySidecarFields)) - .default([]), - entities: z.array(entityEntrySchema).default([]), -}) - -export type MemoryOperations = z.infer - -export type ExistingMemoryContextItem = { - readonly id: string - readonly kind: FluidMemoryKind - readonly abstractL0: string - readonly payloadSummary: string -} - /** Structural output shape only — no domain content. */ -const OUTPUT_SCHEMA_BLOCK = `{ - "indicatorPrefs": [{ - "name": "string", - "aliases": ["string"], - "definition": "string", - "polarity": "higher_better|lower_better|context", - "importance": "core|secondary", - "formulaHint": "string (optional — omit if none)", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }], - "stances": [{ - "statement": "string (the stance text; do not use a name field)", - "scope": "string", - "rationale": "string", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }], - "decisionRules": [{ - "when": "string", - "then": "string", - "priority": "high|medium|low", - "rationale": "string", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }], - "entities": [{ - "name": "string", - "ticker": "string optional", - "aliases": ["string"], - "knowhereDocumentIds": ["only ids listed under REFERENCED DOCUMENT IDS"], - "reason": "string", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } +const CAPTURE_OUTPUT_SCHEMA_BLOCK = `{ + "observations": [{ + "signal": "string — one durable clue about what the USER cares about", + "evidenceQuote": "string — short verbatim USER snippet supporting signal", + "subjectHint": "string optional — topic anchor (metric name / company / topic)", + "confidence": 0.0 }] }` /** - * Illustrative only — kept separate from the main instructions so the model - * does not treat these domain phrases as required vocabulary. - * Finance is the first vertical; add other industry blocks here later if needed. + * Illustrative only — kept separate so the model does not treat these domain + * phrases as required vocabulary. */ const ILLUSTRATIVE_EXAMPLES_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) -These show shape and judgement only. Extract whatever the user actually said; -do not force the conversation into this domain or these metric names. +These show shape and judgement only. Capture whatever the user actually said; +do not force the conversation into this domain. -- indicatorPref: user says a named metric they repeatedly use to judge quality - (shape: name + short definition + polarity + importance). Same idea applies - outside finance (any recurring evaluation metric). -- stance: user states a durable judgement frame that changes how evidence is - weighted (e.g. long-horizon vs short-horizon). -- decisionRule: user states a reusable when → then discipline over their metrics. -- entity: user says they actively track a named company/issuer and why. -- skip: a one-off factual question about a page/number in a document, small talk, - or an assistant suggestion the user did not endorse.` +- User names a recurring evaluation metric → one observation (signal + quote). +- User states a durable judgement frame (e.g. long-horizon) → one observation. +- User states a reusable when → then discipline → one observation. +- User says they actively track a named company → one observation. +- Skip: a one-off factual question about a page/number, small talk, or an + assistant suggestion the user did not endorse.` -/** Domain-agnostic extraction instructions. */ -const MAIN_INSTRUCTIONS_BLOCK = `You maintain a user's FLUID MEMORY: durable insights about how this user thinks, extracted from their conversation with an AI analyst. +const MAIN_INSTRUCTIONS_BLOCK = `You capture RAW OBSERVATIONS for a user's fluid memory pipeline. -Document facts live elsewhere (crystal memory). Never extract document facts, retrieved numbers, or page content as fluid memory. +These are cheap, high-recall clues about what the USER cares about. A later +distill step will decide final kinds (indicator / rule / stance / entity) and +merge them. Do NOT classify observations into those kinds. Do NOT emit any +kind / type / category field. Do NOT deduplicate. Do NOT invent +create/merge/deprecate operations. -## What to extract +Document facts live elsewhere (crystal memory). Never capture document facts, +retrieved numbers, or page content as observations. -Extract ONLY these four kinds, and ONLY when the turn gives real evidence from the USER: +## What to capture -- indicatorPrefs — recurring metrics or criteria the user uses to evaluate things. - Fields: name, aliases, definition, polarity (higher_better | lower_better | context), - importance (core | secondary), optional formulaHint. -- stances — durable positions that shape how the user weighs evidence. - Fields: statement (required; do not invent a "name" field), scope, rationale. -- decisionRules — reusable when → then disciplines the user stated or clearly endorsed. - Fields: when, then, priority (high | medium | low), rationale. -- entities — named subjects the user is actively tracking. - Fields: name, optional ticker, aliases, reason (required), knowhereDocumentIds - (only from REFERENCED DOCUMENT IDS below; never invent ids). +From the USER turn only, emit zero or more observations when there is real +evidence of a durable, reusable point of concern: + +- signal — one short clue in the user's language (what to remember later). +- evidenceQuote — a short verbatim snippet from the USER turn that supports it. +- subjectHint — optional short topic anchor (metric name, company, topic). Prefer omit when none. +- confidence — 1 only when the user stated it explicitly. ## Language - Keep this instruction set and enum/field names in English. -- Write every free-text value (name, definition, statement, when/then, reason, - abstractL0, overviewL1, aliases the user used, etc.) in the same language the - USER wrote in this turn. Do not translate the user's terms into English unless - the user themselves used English. +- Write every free-text value (signal, evidenceQuote, subjectHint) in the same + language the USER wrote in this turn. Do not translate the user's terms into + English unless the user themselves used English. -## Decision rules +## Judgement -- Extract only durable, reusable insights about the USER. -- Skip one-off questions, document facts, small talk, and assistant claims the user did not endorse. -- For every candidate, choose exactly one op against EXISTING MEMORIES: - - create — genuinely new - - skip — already covered, or too weak/ephemeral - - merge — same insight refined; emit the full merged fields and set targetItemId - - deprecate — user explicitly reversed a stored item; set targetItemId -- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. -- Prefer one record per insight. If a preference already encodes how a metric should be read, do not also invent a near-duplicate decisionRule unless the user stated an explicit when → then action. -- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when the user stated it explicitly. +- Capture only durable, reusable clues about what the USER cares about. +- Skip one-off questions, document facts, small talk, and assistant claims the + user did not endorse. +- Prefer atomic clues: one observation per distinct clue. Do not merge unrelated + ideas into one signal. - Omit optional fields instead of setting them to null. -- If nothing is worth remembering, return all four arrays empty.` +- If nothing is worth capturing, return {"observations": []}.` -export function buildMemoryExtractionPrompt(input: { +export function buildCapturePrompt(input: { readonly userText: string readonly assistantText: string readonly referencedDocumentIds: readonly string[] - readonly existingItems: readonly ExistingMemoryContextItem[] }): string { - const existingBlock = - input.existingItems.length === 0 - ? "(no existing memories yet)" - : input.existingItems - .map( - (item) => - `- [${item.kind}] id=${item.id} :: ${item.abstractL0} :: ${item.payloadSummary}`, - ) - .join("\n") - const documentsBlock = input.referencedDocumentIds.length === 0 ? "(no documents referenced in this turn)" @@ -241,11 +86,7 @@ ${ILLUSTRATIVE_EXAMPLES_BLOCK} ## Output JSON schema (follow exactly; do not invent fields) -${OUTPUT_SCHEMA_BLOCK} - -## EXISTING MEMORIES - -${existingBlock} +${CAPTURE_OUTPUT_SCHEMA_BLOCK} ## REFERENCED DOCUMENT IDS @@ -259,27 +100,3 @@ ${input.userText} [assistant] ${input.assistantText}` } - -export function summarizePayloadForContext( - kind: FluidMemoryKind, - payload: unknown, -): string { - if (!payload || typeof payload !== "object") return "" - const record = payload as Record - switch (kind) { - case "indicator_pref": - return [record.name, record.definition] - .filter((part) => typeof part === "string" && part.length > 0) - .join(" — ") - case "stance": - return typeof record.statement === "string" ? record.statement : "" - case "decision_rule": - return [record.when, record.then] - .filter((part) => typeof part === "string" && part.length > 0) - .join(" => ") - case "entity_of_interest": - return [record.name, record.ticker] - .filter((part) => typeof part === "string" && part.length > 0) - .join(" ") - } -} diff --git a/src/domains/memory/repository.ts b/src/domains/memory/repository.ts index f7f6b794..3d36662c 100644 --- a/src/domains/memory/repository.ts +++ b/src/domains/memory/repository.ts @@ -1,8 +1,9 @@ import "server-only" -import { and, eq, inArray, sql } from "drizzle-orm" +import { and, asc, count, eq, inArray, lt, sql } from "drizzle-orm" import { Effect } from "effect" +import type { CapturedObservation } from "./observation-types" import { toDiffOperation, type ResolvedMemoryOperation } from "./resolve-operations" import { buildMemoryItemTokens } from "./search-index" import type { @@ -10,15 +11,31 @@ import type { FluidMemoryPayload, MemoryDiffOperation, } from "./types" -import { DbClient } from "@/infrastructure/db" +import { DbClient, type Db } from "@/infrastructure/db" import { fluidMemoryItems, fluidMemoryTokens, + fluidObservations, memoryDiffs, type FluidMemoryItem, + type FluidObservation, type NewFluidMemoryToken, } from "@/infrastructure/db/schema" +export type InsertObservationsInput = { + readonly workspaceId: string + readonly sourceMessageId: string | null + readonly referencedDocumentIds: readonly string[] + readonly observations: readonly CapturedObservation[] +} + +export type ApplyDistillBatchInput = { + readonly workspaceId: string + readonly sourceMessageId: string | null + readonly operations: readonly ResolvedMemoryOperation[] + readonly observationIds: readonly string[] +} + type MemoryRepository = { readonly findDedupCandidatesEffect: ( workspaceId: string, @@ -26,15 +43,35 @@ type MemoryRepository = { tokens: readonly string[], limit: number, ) => Effect.Effect - readonly applyOperationsEffect: ( + readonly insertObservationsEffect: ( + input: InsertObservationsInput, + ) => Effect.Effect + readonly countPendingObservationsEffect: ( workspaceId: string, - sourceMessageId: string | null, - operations: readonly ResolvedMemoryOperation[], - ) => Effect.Effect + ) => Effect.Effect + readonly listPendingObservationsEffect: ( + workspaceId: string, + limit: number, + ) => Effect.Effect + readonly applyDistillBatchEffect: ( + input: ApplyDistillBatchInput, + ) => Effect.Effect< + { + readonly diffs: readonly MemoryDiffOperation[] + readonly consumedCount: number + }, + never, + DbClient + > + readonly deleteExpiredConsumedObservationsEffect: ( + olderThan: Date, + ) => Effect.Effect } type RawRowsResult = readonly Row[] | { readonly rows: readonly Row[] } +type TxClient = Parameters[0]>[0] + /** * Retrieve the most lexically-similar active items of one kind, ranked by * idf-weighted token overlap computed entirely in SQL. Common tokens (high @@ -92,145 +129,272 @@ const findDedupCandidatesEffect: MemoryRepository["findDedupCandidatesEffect"] = }) }) -const applyOperationsEffect: MemoryRepository["applyOperationsEffect"] = ( - workspaceId, - sourceMessageId, - operations, +/** + * Append-only write of coarse-capture clues. Never touches fluid_memory_items. + * Empty input is a no-op (returns []). Status is always `pending`. + */ +const insertObservationsEffect: MemoryRepository["insertObservationsEffect"] = ( + input, +) => + Effect.gen(function* () { + const db = yield* DbClient + if (input.observations.length === 0) return [] + + const documentIds = [...input.referencedDocumentIds] + return yield* Effect.promise(() => + db + .insert(fluidObservations) + .values( + input.observations.map((observation) => ({ + workspaceId: input.workspaceId, + sourceMessageId: input.sourceMessageId, + signal: observation.signal, + evidenceQuote: observation.evidenceQuote, + subjectHint: observation.subjectHint ?? null, + referencedDocumentIds: documentIds, + confidence: observation.confidence, + status: "pending", + })), + ) + .returning(), + ) + }) + +const countPendingObservationsEffect: MemoryRepository["countPendingObservationsEffect"] = + (workspaceId) => + Effect.gen(function* () { + const db = yield* DbClient + const rows = yield* Effect.promise(() => + db + .select({ value: count() }) + .from(fluidObservations) + .where( + and( + eq(fluidObservations.workspaceId, workspaceId), + eq(fluidObservations.status, "pending"), + ), + ), + ) + return Number(rows[0]?.value ?? 0) + }) + +/** + * Oldest-first pending batch for distill. Concurrency across distill runs for + * the same workspace is primarily gated by trigger cooldown + bucketed + * workflowRunId; consume below is conditional on status still being pending. + */ +const listPendingObservationsEffect: MemoryRepository["listPendingObservationsEffect"] = + (workspaceId, limit) => + Effect.gen(function* () { + const db = yield* DbClient + if (limit <= 0) return [] + return yield* Effect.promise(() => + db + .select() + .from(fluidObservations) + .where( + and( + eq(fluidObservations.workspaceId, workspaceId), + eq(fluidObservations.status, "pending"), + ), + ) + .orderBy(asc(fluidObservations.createdAt)) + .limit(limit), + ) + }) + +const applyDistillBatchEffect: MemoryRepository["applyDistillBatchEffect"] = ( + input, ) => Effect.gen(function* () { const db = yield* DbClient return yield* Effect.promise(() => db.transaction(async (tx) => { - const diffOperations: MemoryDiffOperation[] = [] - - for (const operation of operations) { - switch (operation.op) { - case "create": { - const [inserted] = await tx - .insert(fluidMemoryItems) - .values({ - workspaceId, - kind: operation.kind, - payload: operation.payload, - abstractL0: operation.abstractL0, - overviewL1: operation.overviewL1, - sourceMessageId, - confidence: operation.confidence, - status: "active", - }) - .returning() - if (inserted?.id) { - const tokenRows = tokenRowsFor( - workspaceId, - inserted.id, - operation.kind, - operation.payload, - ) - if (tokenRows.length > 0) { - await tx.insert(fluidMemoryTokens).values(tokenRows) - } - } - diffOperations.push(toDiffOperation(operation, inserted?.id)) - break - } - case "merge": { - const [updated] = await tx - .update(fluidMemoryItems) - .set({ - payload: operation.payload, - abstractL0: operation.abstractL0, - overviewL1: operation.overviewL1, - confidence: operation.confidence, - sourceMessageId, - version: sql`${fluidMemoryItems.version} + 1`, - updatedAt: sql`now()`, - }) - .where( - and( - eq(fluidMemoryItems.id, operation.targetItemId), - eq(fluidMemoryItems.status, "active"), - ), - ) - .returning({ id: fluidMemoryItems.id }) - if (updated) { - await tx - .delete(fluidMemoryTokens) - .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) - const tokenRows = tokenRowsFor( - workspaceId, - operation.targetItemId, - operation.kind, - operation.payload, - ) - if (tokenRows.length > 0) { - await tx.insert(fluidMemoryTokens).values(tokenRows) - } - } - diffOperations.push( - updated - ? toDiffOperation(operation) - : { - op: "skip", - kind: operation.kind, - summary: operation.summary, - reason: "merge target no longer active", - }, - ) - break - } - case "deprecate": { - const [updated] = await tx - .update(fluidMemoryItems) - .set({ - status: "deprecated", - updatedAt: sql`now()`, - }) - .where( - and( - eq(fluidMemoryItems.id, operation.targetItemId), - eq(fluidMemoryItems.status, "active"), - ), - ) - .returning({ id: fluidMemoryItems.id }) - if (updated) { - await tx - .delete(fluidMemoryTokens) - .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) - } - diffOperations.push( - updated - ? toDiffOperation(operation) - : { - op: "skip", - kind: operation.kind, - summary: operation.summary, - reason: "deprecate target no longer active", - }, - ) - break - } - case "skip": - diffOperations.push(toDiffOperation(operation)) - break - } + const diffs = await writeOperations( + tx, + input.workspaceId, + input.sourceMessageId, + input.operations, + ) + let consumedCount = 0 + if (input.observationIds.length > 0) { + const consumed = await tx + .update(fluidObservations) + .set({ + status: "consumed", + consumedAt: sql`now()`, + }) + .where( + and( + eq(fluidObservations.workspaceId, input.workspaceId), + eq(fluidObservations.status, "pending"), + inArray(fluidObservations.id, [...input.observationIds]), + ), + ) + .returning({ id: fluidObservations.id }) + consumedCount = consumed.length } + return { diffs, consumedCount } + }), + ) + }) + +const deleteExpiredConsumedObservationsEffect: MemoryRepository["deleteExpiredConsumedObservationsEffect"] = + (olderThan) => + Effect.gen(function* () { + const db = yield* DbClient + const deleted = yield* Effect.promise(() => + db + .delete(fluidObservations) + .where( + and( + eq(fluidObservations.status, "consumed"), + lt(fluidObservations.createdAt, olderThan), + ), + ) + .returning({ id: fluidObservations.id }), + ) + return deleted.length + }) + +export const memoryRepository: MemoryRepository = { + findDedupCandidatesEffect, + insertObservationsEffect, + countPendingObservationsEffect, + listPendingObservationsEffect, + applyDistillBatchEffect, + deleteExpiredConsumedObservationsEffect, +} + +async function writeOperations( + tx: TxClient, + workspaceId: string, + sourceMessageId: string | null, + operations: readonly ResolvedMemoryOperation[], +): Promise { + const diffOperations: MemoryDiffOperation[] = [] - if (diffOperations.length > 0) { - await tx.insert(memoryDiffs).values({ + for (const operation of operations) { + switch (operation.op) { + case "create": { + const [inserted] = await tx + .insert(fluidMemoryItems) + .values({ workspaceId, + kind: operation.kind, + payload: operation.payload, + abstractL0: operation.abstractL0, + overviewL1: operation.overviewL1, sourceMessageId, - operations: [...diffOperations], + confidence: operation.confidence, + status: "active", }) + .returning() + if (inserted?.id) { + const tokenRows = tokenRowsFor( + workspaceId, + inserted.id, + operation.kind, + operation.payload, + ) + if (tokenRows.length > 0) { + await tx.insert(fluidMemoryTokens).values(tokenRows) + } + } + diffOperations.push(toDiffOperation(operation, inserted?.id)) + break + } + case "merge": { + const [updated] = await tx + .update(fluidMemoryItems) + .set({ + payload: operation.payload, + abstractL0: operation.abstractL0, + overviewL1: operation.overviewL1, + confidence: operation.confidence, + sourceMessageId, + version: sql`${fluidMemoryItems.version} + 1`, + updatedAt: sql`now()`, + }) + .where( + and( + eq(fluidMemoryItems.id, operation.targetItemId), + eq(fluidMemoryItems.status, "active"), + ), + ) + .returning({ id: fluidMemoryItems.id }) + if (updated) { + await tx + .delete(fluidMemoryTokens) + .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) + const tokenRows = tokenRowsFor( + workspaceId, + operation.targetItemId, + operation.kind, + operation.payload, + ) + if (tokenRows.length > 0) { + await tx.insert(fluidMemoryTokens).values(tokenRows) + } } + diffOperations.push( + updated + ? toDiffOperation(operation) + : { + op: "skip", + kind: operation.kind, + summary: operation.summary, + reason: "merge target no longer active", + }, + ) + break + } + case "deprecate": { + const [updated] = await tx + .update(fluidMemoryItems) + .set({ + status: "deprecated", + updatedAt: sql`now()`, + }) + .where( + and( + eq(fluidMemoryItems.id, operation.targetItemId), + eq(fluidMemoryItems.status, "active"), + ), + ) + .returning({ id: fluidMemoryItems.id }) + if (updated) { + await tx + .delete(fluidMemoryTokens) + .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) + } + diffOperations.push( + updated + ? toDiffOperation(operation) + : { + op: "skip", + kind: operation.kind, + summary: operation.summary, + reason: "deprecate target no longer active", + }, + ) + break + } + case "skip": + diffOperations.push(toDiffOperation(operation)) + break + } + } - return diffOperations - }), - ) - }) + if (diffOperations.length > 0) { + await tx.insert(memoryDiffs).values({ + workspaceId, + sourceMessageId, + operations: [...diffOperations], + }) + } -export const memoryRepository: MemoryRepository = { - findDedupCandidatesEffect, - applyOperationsEffect, + return diffOperations } function tokenRowsFor( diff --git a/src/domains/memory/resolve-operations.test.ts b/src/domains/memory/resolve-operations.test.ts index 9764ccfd..403a79a8 100644 --- a/src/domains/memory/resolve-operations.test.ts +++ b/src/domains/memory/resolve-operations.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" -import type { MemoryOperations } from "./prompts" +import type { MemoryOperations } from "./resolve-operations" import { resolveMemoryOperations, toDiffOperation, diff --git a/src/domains/memory/resolve-operations.ts b/src/domains/memory/resolve-operations.ts index ee1e6a42..833b9553 100644 --- a/src/domains/memory/resolve-operations.ts +++ b/src/domains/memory/resolve-operations.ts @@ -1,5 +1,5 @@ +import type { MemoryOperations } from "./distill-types" import { buildMemoryItemTokens } from "./search-index" -import type { MemoryOperations } from "./prompts" import { parseFluidMemoryPayload, type FluidMemoryKind, @@ -7,6 +7,8 @@ import { type MemoryDiffOperation, } from "./types" +export type { MemoryOperations } from "./distill-types" + /** * Pure normalization from raw LLM operations to repository-ready * operations. The LLM output already passed zod validation; this layer @@ -14,12 +16,14 @@ import { * - merge/deprecate must target an existing active item of the same kind * (otherwise downgraded to skip — conservative, never fabricates) * - entity knowhereDocumentIds are intersected with the document ids - * actually referenced in the turn (the model cannot invent provenance) + * allowed for the batch (the model cannot invent provenance) * - create ignores any targetItemId the model may have emitted * - create/merge payloads must yield at least one lexical token, otherwise * the item could never be retrieved for later dedup * - merge unions aliases (and entity document ids) with the target so * prior search terms / provenance are not wiped by a partial rewrite + * + * Used by distill (not by per-turn capture). Capture only writes observations. */ export type ResolvedMemoryOperation = @@ -65,17 +69,6 @@ export type ExistingMemoryItemRef = { readonly payload?: unknown } -type CandidateEntry = { - readonly abstractL0: string - readonly overviewL1: string - readonly confidence: number - readonly decision: { - readonly op: "create" | "skip" | "merge" | "deprecate" - readonly targetItemId?: string - readonly reason?: string - } -} - const kindToArrayKey = { indicator_pref: "indicatorPrefs", stance: "stances", @@ -98,8 +91,7 @@ export function resolveMemoryOperations(input: { const resolved: ResolvedMemoryOperation[] = [] for (const kind of Object.keys(kindToArrayKey) as FluidMemoryKind[]) { - const entries = input.operations[kindToArrayKey[kind]] as readonly (CandidateEntry & - Record)[] + const entries = input.operations[kindToArrayKey[kind]] for (const entry of entries) { const summary = entry.abstractL0 @@ -132,7 +124,11 @@ export function resolveMemoryOperations(input: { }) continue } - const mergePayload = toPayload(kind, entry, allowedDocumentIds) + const mergePayload = toPayload( + kind, + entry as Record, + allowedDocumentIds, + ) if (!mergePayload) { resolved.push({ op: "skip", @@ -170,7 +166,11 @@ export function resolveMemoryOperations(input: { continue } - const createPayload = toPayload(kind, entry, allowedDocumentIds) + const createPayload = toPayload( + kind, + entry as Record, + allowedDocumentIds, + ) if (!createPayload) { resolved.push({ op: "skip", @@ -241,7 +241,7 @@ function isIndexable(kind: FluidMemoryKind, payload: FluidMemoryPayload): boolea } /** - * Merge replaces the stored payload, but the model only sees this turn. + * Merge replaces the stored payload, but the model only sees this batch. * Union aliases (and entity document ids) with the target so earlier search * terms / provenance survive a partial rewrite. */ diff --git a/src/domains/memory/service.ts b/src/domains/memory/service.ts index da772f65..41b8b5af 100644 --- a/src/domains/memory/service.ts +++ b/src/domains/memory/service.ts @@ -1,10 +1,16 @@ import "server-only" import { databaseRuntime } from "@/domains/workspace/database-runtime" -import { memoryRepository } from "./repository" -import type { ResolvedMemoryOperation } from "./resolve-operations" +import { + memoryRepository, + type ApplyDistillBatchInput, + type InsertObservationsInput, +} from "./repository" import type { FluidMemoryKind, MemoryDiffOperation } from "./types" -import type { FluidMemoryItem } from "@/infrastructure/db/schema" +import type { + FluidMemoryItem, + FluidObservation, +} from "@/infrastructure/db/schema" type MemoryService = { readonly findDedupCandidates: ( @@ -13,11 +19,21 @@ type MemoryService = { tokens: readonly string[], limit: number, ) => Promise - readonly applyOperations: ( + readonly insertObservations: ( + input: InsertObservationsInput, + ) => Promise + readonly countPendingObservations: (workspaceId: string) => Promise + readonly listPendingObservations: ( workspaceId: string, - sourceMessageId: string | null, - operations: readonly ResolvedMemoryOperation[], - ) => Promise + limit: number, + ) => Promise + readonly applyDistillBatch: (input: ApplyDistillBatchInput) => Promise<{ + readonly diffs: readonly MemoryDiffOperation[] + readonly consumedCount: number + }> + readonly deleteExpiredConsumedObservations: ( + olderThan: Date, + ) => Promise } const findDedupCandidates: MemoryService["findDedupCandidates"] = ( @@ -35,20 +51,38 @@ const findDedupCandidates: MemoryService["findDedupCandidates"] = ( ), ) -const applyOperations: MemoryService["applyOperations"] = ( +const insertObservations: MemoryService["insertObservations"] = (input) => + databaseRuntime.runPromise(memoryRepository.insertObservationsEffect(input)) + +const countPendingObservations: MemoryService["countPendingObservations"] = ( workspaceId, - sourceMessageId, - operations, ) => databaseRuntime.runPromise( - memoryRepository.applyOperationsEffect( - workspaceId, - sourceMessageId, - operations, - ), + memoryRepository.countPendingObservationsEffect(workspaceId), + ) + +const listPendingObservations: MemoryService["listPendingObservations"] = ( + workspaceId, + limit, +) => + databaseRuntime.runPromise( + memoryRepository.listPendingObservationsEffect(workspaceId, limit), ) +const applyDistillBatch: MemoryService["applyDistillBatch"] = (input) => + databaseRuntime.runPromise(memoryRepository.applyDistillBatchEffect(input)) + +const deleteExpiredConsumedObservations: MemoryService["deleteExpiredConsumedObservations"] = + (olderThan) => + databaseRuntime.runPromise( + memoryRepository.deleteExpiredConsumedObservationsEffect(olderThan), + ) + export const memoryService: MemoryService = { findDedupCandidates, - applyOperations, + insertObservations, + countPendingObservations, + listPendingObservations, + applyDistillBatch, + deleteExpiredConsumedObservations, } diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index a3c98054..cb767a02 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -476,3 +476,53 @@ export const memoryDiffs = pgTable( export type MemoryDiff = typeof memoryDiffs.$inferSelect; export type NewMemoryDiff = typeof memoryDiffs.$inferInsert; + +/** + * Append-only raw observation layer for fluid memory (L2 evidence). + * + * Each chat turn may write zero or more pending rows here via cheap capture. + * A later distill job consumes a batch, upserts typed items into + * `fluid_memory_items`, and marks these rows `consumed`. Capture never writes + * the distilled layer; distill is the only writer of permanent memory. + * + * Capture stores points of concern only — no early kind classification. + * `subject_hint` is an optional topic anchor for later clustering; distill + * owns the final kind and merge decision. + */ +export const fluidObservations = pgTable( + "fluid_observations", + { + id: uuid("id").primaryKey().defaultRandom(), + workspaceId: uuid("workspace_id") + .notNull() + .references(() => workspaces.id, { onDelete: "cascade" }), + sourceMessageId: uuid("source_message_id").references( + () => chatMessages.id, + { onDelete: "set null" }, + ), + signal: text("signal").notNull(), + evidenceQuote: text("evidence_quote").notNull(), + subjectHint: text("subject_hint"), + referencedDocumentIds: jsonb("referenced_document_ids") + .$type() + .notNull() + .default(sql`'[]'::jsonb`), + confidence: doublePrecision("confidence").notNull(), + status: text("status").notNull().default("pending"), + createdAt: timestamp("created_at", { withTimezone: true }) + .notNull() + .defaultNow(), + consumedAt: timestamp("consumed_at", { withTimezone: true }), + }, + (t) => [ + // Distill scan: pending rows for a workspace in capture order. + index("fluid_observations_workspace_status_created_idx").on( + t.workspaceId, + t.status, + t.createdAt, + ), + ], +); + +export type FluidObservation = typeof fluidObservations.$inferSelect; +export type NewFluidObservation = typeof fluidObservations.$inferInsert; From 823356c3cf842c73e247d31dd534b7990d5e980d Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Thu, 10 Sep 2026 10:49:03 +0800 Subject: [PATCH 07/11] feat(memory): enhance memory search functionality and integrate into agent harness - Added a new memory search tool to retrieve insights from fluid memory, allowing for more context-aware responses. - Updated the agent harness to utilize memory tools, ensuring that memory searches are prioritized before document retrieval. - Introduced memory citations in output manifests to track references from memory searches. - Enhanced tests to validate the new memory search behavior and its integration with existing functionalities. This update improves the agent's ability to leverage past interactions for more relevant and informed responses. --- drizzle/0016_wealthy_matthew_murdock.sql | 13 + drizzle/meta/0016_snapshot.json | 1550 +++++++++++++++++ drizzle/meta/_journal.json | 7 + scripts/guanxin-case/constants.ts | 8 + scripts/guanxin-case/ensure-workspace.mts | 21 + scripts/guanxin-case/sample-pilot.py | 91 + src/agent-harness/index.ts | 1 + src/agent-harness/memory-text.test.ts | 38 + src/agent-harness/memory-text.ts | 93 + src/agent-harness/runtime.test.ts | 190 +- src/agent-harness/runtime.ts | 179 +- src/agent-harness/types.ts | 38 + src/domains/chat/citations.test.ts | 15 + src/domains/chat/citations.ts | 1 + src/domains/chat/commit-turn.test.ts | 138 ++ src/domains/chat/commit-turn.ts | 131 ++ src/domains/chat/index.test.ts | 23 +- src/domains/chat/memory-tools.test.ts | 104 ++ src/domains/chat/memory-tools.ts | 66 + src/domains/chat/prompt.ts | 5 + src/domains/chat/route-answer.ts | 24 +- src/domains/chat/route-service.test.ts | 2 +- src/domains/chat/service.test.ts | 34 + src/domains/chat/service.ts | 2 +- src/domains/chat/types.ts | 1 + src/domains/memory/decay-candidates.test.ts | 69 + src/domains/memory/decay-candidates.ts | 64 + src/domains/memory/repository.ts | 84 +- src/domains/memory/resolve-operations.test.ts | 4 +- src/domains/memory/service.ts | 58 + src/domains/memory/types.ts | 11 + .../retrieval-activation/decay-score.test.ts | 98 ++ .../retrieval-activation/decay-score.ts | 66 + .../retrieval-activation/repository.test.ts | 138 ++ .../retrieval-activation/repository.ts | 119 ++ src/domains/retrieval-activation/service.ts | 45 + src/domains/retrieval-activation/types.ts | 17 + src/infrastructure/db/schema.ts | 60 +- 38 files changed, 3556 insertions(+), 52 deletions(-) create mode 100644 drizzle/0016_wealthy_matthew_murdock.sql create mode 100644 drizzle/meta/0016_snapshot.json create mode 100644 scripts/guanxin-case/constants.ts create mode 100644 scripts/guanxin-case/ensure-workspace.mts create mode 100644 scripts/guanxin-case/sample-pilot.py create mode 100644 src/agent-harness/memory-text.test.ts create mode 100644 src/agent-harness/memory-text.ts create mode 100644 src/domains/chat/commit-turn.test.ts create mode 100644 src/domains/chat/commit-turn.ts create mode 100644 src/domains/chat/memory-tools.test.ts create mode 100644 src/domains/chat/memory-tools.ts create mode 100644 src/domains/memory/decay-candidates.test.ts create mode 100644 src/domains/memory/decay-candidates.ts create mode 100644 src/domains/retrieval-activation/decay-score.test.ts create mode 100644 src/domains/retrieval-activation/decay-score.ts create mode 100644 src/domains/retrieval-activation/repository.test.ts create mode 100644 src/domains/retrieval-activation/repository.ts create mode 100644 src/domains/retrieval-activation/service.ts create mode 100644 src/domains/retrieval-activation/types.ts diff --git a/drizzle/0016_wealthy_matthew_murdock.sql b/drizzle/0016_wealthy_matthew_murdock.sql new file mode 100644 index 00000000..4d7a57ec --- /dev/null +++ b/drizzle/0016_wealthy_matthew_murdock.sql @@ -0,0 +1,13 @@ +CREATE TABLE "retrieval_activations" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "unit_type" text NOT NULL, + "unit_ref" text NOT NULL, + "activation_count" integer DEFAULT 0 NOT NULL, + "last_activated_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "fluid_memory_items" ADD COLUMN "deactivation_reason" text;--> statement-breakpoint +ALTER TABLE "retrieval_activations" ADD CONSTRAINT "retrieval_activations_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "retrieval_activations_unit_idx" ON "retrieval_activations" USING btree ("workspace_id","unit_type","unit_ref"); \ No newline at end of file diff --git a/drizzle/meta/0016_snapshot.json b/drizzle/meta/0016_snapshot.json new file mode 100644 index 00000000..77ef531c --- /dev/null +++ b/drizzle/meta/0016_snapshot.json @@ -0,0 +1,1550 @@ +{ + "id": "55d73dc7-9826-4cb5-9bb6-3506b8fa338a", + "prevId": "be8deb1f-3ec8-4304-85d0-41403629970c", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_memory_items": { + "name": "fluid_memory_items", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "abstract_l0": { + "name": "abstract_l0", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "overview_l1": { + "name": "overview_l1", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deactivation_reason": { + "name": "deactivation_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "fluid_memory_items_workspace_status_idx": { + "name": "fluid_memory_items_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fluid_memory_items_workspace_kind_idx": { + "name": "fluid_memory_items_workspace_kind_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_memory_items_workspace_id_workspaces_id_fk": { + "name": "fluid_memory_items_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_memory_items_source_message_id_chat_messages_id_fk": { + "name": "fluid_memory_items_source_message_id_chat_messages_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_memory_tokens": { + "name": "fluid_memory_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "item_id": { + "name": "item_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "frequency": { + "name": "frequency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + } + }, + "indexes": { + "fluid_memory_tokens_lookup_idx": { + "name": "fluid_memory_tokens_lookup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fluid_memory_tokens_item_idx": { + "name": "fluid_memory_tokens_item_idx", + "columns": [ + { + "expression": "item_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_memory_tokens_workspace_id_workspaces_id_fk": { + "name": "fluid_memory_tokens_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_memory_tokens", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_memory_tokens_item_id_fluid_memory_items_id_fk": { + "name": "fluid_memory_tokens_item_id_fluid_memory_items_id_fk", + "tableFrom": "fluid_memory_tokens", + "tableTo": "fluid_memory_items", + "columnsFrom": [ + "item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_observations": { + "name": "fluid_observations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "signal": { + "name": "signal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "evidence_quote": { + "name": "evidence_quote", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "subject_hint": { + "name": "subject_hint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "referenced_document_ids": { + "name": "referenced_document_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "confidence": { + "name": "confidence", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "fluid_observations_workspace_status_created_idx": { + "name": "fluid_observations_workspace_status_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_observations_workspace_id_workspaces_id_fk": { + "name": "fluid_observations_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_observations", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_observations_source_message_id_chat_messages_id_fk": { + "name": "fluid_observations_source_message_id_chat_messages_id_fk", + "tableFrom": "fluid_observations", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_diffs": { + "name": "memory_diffs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memory_diffs_workspace_created_idx": { + "name": "memory_diffs_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_diffs_workspace_id_workspaces_id_fk": { + "name": "memory_diffs_workspace_id_workspaces_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memory_diffs_source_message_id_chat_messages_id_fk": { + "name": "memory_diffs_source_message_id_chat_messages_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parsed_document_sync_leases": { + "name": "parsed_document_sync_leases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_token": { + "name": "lease_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_reason": { + "name": "release_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parsed_document_sync_leases_token_idx": { + "name": "parsed_document_sync_leases_token_idx", + "columns": [ + { + "expression": "lease_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_active_idx": { + "name": "parsed_document_sync_leases_active_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_workspace_active_idx": { + "name": "parsed_document_sync_leases_workspace_active_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_document_active_idx": { + "name": "parsed_document_sync_leases_document_active_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parsed_document_sync_leases_workspace_id_workspaces_id_fk": { + "name": "parsed_document_sync_leases_workspace_id_workspaces_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parsed_document_sync_leases_source_id_sources_id_fk": { + "name": "parsed_document_sync_leases_source_id_sources_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.retrieval_activations": { + "name": "retrieval_activations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "unit_type": { + "name": "unit_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "unit_ref": { + "name": "unit_ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activation_count": { + "name": "activation_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_activated_at": { + "name": "last_activated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "retrieval_activations_unit_idx": { + "name": "retrieval_activations_unit_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "unit_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "unit_ref", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "retrieval_activations_workspace_id_workspaces_id_fk": { + "name": "retrieval_activations_workspace_id_workspaces_id_fk", + "tableFrom": "retrieval_activations", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 32ea955d..a8f7671e 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -113,6 +113,13 @@ "when": 1788418416499, "tag": "0015_bumpy_vulcan", "breakpoints": true + }, + { + "idx": 16, + "version": "7", + "when": 1788952172906, + "tag": "0016_wealthy_matthew_murdock", + "breakpoints": true } ] } \ No newline at end of file diff --git a/scripts/guanxin-case/constants.ts b/scripts/guanxin-case/constants.ts new file mode 100644 index 00000000..d4f60992 --- /dev/null +++ b/scripts/guanxin-case/constants.ts @@ -0,0 +1,8 @@ +/** + * Dedicated Notebook workspace for the 观心 cardiovascular case. + * workspaces 表没有 title 列，用稳定 userId 标识。 + */ +export const GUANXIN_CASE_USER_ID = "case:guanxin-cardiovascular" + +/** 方案约定的 pilot 规模（20–30）按 sheet×难度 6 层均分。 */ +export const PILOT_CASES_PER_STRATUM = 4 diff --git a/scripts/guanxin-case/ensure-workspace.mts b/scripts/guanxin-case/ensure-workspace.mts new file mode 100644 index 00000000..963bca61 --- /dev/null +++ b/scripts/guanxin-case/ensure-workspace.mts @@ -0,0 +1,21 @@ +/** + * Ensure the dedicated 观心 case workspace exists in the Notebook database. + * Requires DATABASE_URL (Notebook Neon/Postgres), not the Knowhere eval DSN. + * + * DATABASE_URL=... node --experimental-strip-types scripts/guanxin-case/ensure-workspace.mts + */ +import { workspaceService } from "../../src/domains/workspace/service.ts" +import { GUANXIN_CASE_USER_ID } from "./constants.ts" + +const workspace = await workspaceService.ensureWorkspace(GUANXIN_CASE_USER_ID) +console.log( + JSON.stringify( + { + userId: workspace.userId, + workspaceId: workspace.id, + namespace: workspace.namespace, + }, + null, + 2, + ), +) diff --git a/scripts/guanxin-case/sample-pilot.py b/scripts/guanxin-case/sample-pilot.py new file mode 100644 index 00000000..a743d4da --- /dev/null +++ b/scripts/guanxin-case/sample-pilot.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Stratified pilot sample from 观心 knowhere自测集.xlsx. + +Takes the first PILOT_CASES_PER_STRATUM rows (by seq_id) from each +(sheet, 难度) bucket. Does not call Knowhere or write the case workspace. +""" +from __future__ import annotations + +import json +import sys +from collections import defaultdict +from pathlib import Path + +import openpyxl + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_XLSX = Path( + "/Users/wuchengke/Desktop/skills-coding/观心2.0-RAG-v1.1-demo/knowhere自测集.xlsx" +) +DEFAULT_OUT = ROOT / ".tmp" / "guanxin-pilot-cases.json" +PILOT_CASES_PER_STRATUM = 4 + + +def load_rows(xlsx: Path) -> list[dict]: + wb = openpyxl.load_workbook(xlsx, read_only=True, data_only=True) + rows: list[dict] = [] + try: + for sheet_name in wb.sheetnames: + sheet_rows = list(wb[sheet_name].iter_rows(values_only=True)) + if not sheet_rows: + continue + header = [ + str(h).strip() if h is not None else f"col{i}" + for i, h in enumerate(sheet_rows[0]) + ] + for raw in sheet_rows[1:]: + if not raw or raw[0] in (None, ""): + continue + item = { + header[i]: (raw[i] if i < len(raw) else None) + for i in range(len(header)) + } + query = str(item.get("具体query") or "").strip() + if not query: + continue + rows.append( + { + "sheet": sheet_name, + "seq_id": str(item.get("seq_id") or ""), + "query": query, + "disease": str(item.get("具体疾病名称") or "").strip(), + "scene": str(item.get("应用场景-考察能力") or "").strip(), + "difficulty": str(item.get("难度") or "").strip(), + "input_type": str(item.get("输入类型") or "").strip(), + } + ) + finally: + wb.close() + return rows + + +def sample(rows: list[dict]) -> list[dict]: + buckets: dict[tuple[str, str], list[dict]] = defaultdict(list) + for row in rows: + buckets[(row["sheet"], row["difficulty"] or "?")].append(row) + picked: list[dict] = [] + for key in sorted(buckets): + group = sorted(buckets[key], key=lambda row: row["seq_id"]) + picked.extend(group[:PILOT_CASES_PER_STRATUM]) + return sorted(picked, key=lambda row: (row["sheet"], row["seq_id"])) + + +def main() -> None: + xlsx = Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_XLSX + out = Path(sys.argv[2]) if len(sys.argv) > 2 else DEFAULT_OUT + rows = load_rows(xlsx) + picked = sample(rows) + out.parent.mkdir(parents=True, exist_ok=True) + payload = { + "xlsx": str(xlsx), + "per_stratum": PILOT_CASES_PER_STRATUM, + "source_count": len(rows), + "pilot_count": len(picked), + "cases": picked, + } + out.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(f"wrote {len(picked)} / {len(rows)} -> {out}") + + +if __name__ == "__main__": + main() diff --git a/src/agent-harness/index.ts b/src/agent-harness/index.ts index a2666bd4..40e843bb 100644 --- a/src/agent-harness/index.ts +++ b/src/agent-harness/index.ts @@ -1,5 +1,6 @@ export * from "./image-highlights" export * from "./ledger" export * from "./knowhere-text" +export * from "./memory-text" export * from "./runtime" export * from "./types" diff --git a/src/agent-harness/memory-text.test.ts b/src/agent-harness/memory-text.test.ts new file mode 100644 index 00000000..aa1699de --- /dev/null +++ b/src/agent-harness/memory-text.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest" + +import { memoryToolText } from "./memory-text" + +describe("memoryToolText", () => { + it("formats search results with memory refs and stored summaries", () => { + const text = memoryToolText.formatSearch({ + query: "毛利率", + items: [ + { + ref: "mem:1", + itemId: "item_1", + kind: "stance", + abstractL0: "关注毛利率下滑", + overviewL1: "用户把毛利率当作核心观察指标。", + }, + ], + }) + + expect(text).toContain('') + expect(text).toContain('query="毛利率"') + expect(text).toContain('ref="mem:1"') + expect(text).toContain('itemId="item_1"') + expect(text).toContain('kind="stance"') + expect(text).toContain("关注毛利率下滑") + expect(text).toContain("用户把毛利率当作核心观察指标。") + }) + + it("formats an empty search without inventing items", () => { + const text = memoryToolText.formatSearch({ + query: "unknown", + items: [], + }) + + expect(text).toContain('resultCount="0"') + expect(text).not.toContain("", + ].join("\n") + }, +} as const + +function formatMemoryItems(response: MemorySearchResponse): string { + if (response.items.length === 0) return "" + + return [ + "", + ...response.items.map((item) => + [ + formatOpenTag("item", { + ref: item.ref, + itemId: item.itemId, + kind: item.kind, + }), + formatTextTag("abstract_l0", item.abstractL0), + formatTextTag("overview_l1", item.overviewL1), + "", + ].join("\n"), + ), + "", + ].join("\n") +} + +function wrapMemoryBlock( + operation: MemoryOperation, + parts: readonly string[], +): string { + return [ + formatOpenTag("memory", { operation, status: "ok" }), + ...parts.filter((part) => part.trim().length > 0), + "", + ].join("\n") +} + +function formatTextTag(tagName: string, value: string): string { + return [`<${tagName}>`, value, ``].join("\n") +} + +function formatTag( + tagName: string, + attrs: Readonly>, +): string { + return `${formatOpenTag(tagName, attrs)}` +} + +function formatOpenTag( + tagName: string, + attrs: Readonly>, +): string { + const serializedAttrs = Object.entries(attrs) + .filter((entry): entry is [string, string] => typeof entry[1] === "string") + .map(([key, value]) => `${key}="${escapeAttribute(value)}"`) + .join(" ") + return serializedAttrs ? `<${tagName} ${serializedAttrs}>` : `<${tagName}>` +} + +function escapeAttribute(value: string): string { + return value + .replaceAll("&", "&") + .replaceAll('"', """) + .replaceAll("<", "<") + .replaceAll(">", ">") +} diff --git a/src/agent-harness/runtime.test.ts b/src/agent-harness/runtime.test.ts index 74e7ef71..c2066c3d 100644 --- a/src/agent-harness/runtime.test.ts +++ b/src/agent-harness/runtime.test.ts @@ -16,10 +16,23 @@ import type { ImageInspectionRequest, IntentFrame, KnowhereToolRuntime, + MemoryToolRuntime, OutputManifest, } from "./types" describe("agent harness runtime", () => { + it("tells the agent to search fluid memory first and not treat every question as document retrieval", () => { + const prompt = buildHarnessSystemPrompt(makeTurnInput()) + + expect(prompt).toContain("Call memory_search first") + expect(prompt).toContain( + "Call knowhere_search only when memory is insufficient and groundingPolicy requires citing source documents", + ) + expect(prompt).toContain( + "Do not treat every question as a document-retrieval task", + ) + }) + it("keeps KNOWHERE as an evidence provider instead of exposing internal navigation", () => { const prompt = buildHarnessSystemPrompt(makeTurnInput()) @@ -72,6 +85,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(query), recentTurns: [], }) @@ -147,6 +161,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(query), recentTurns: [], }) @@ -171,6 +186,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -203,6 +219,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -244,6 +261,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -274,6 +292,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -330,6 +349,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -435,6 +455,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -451,6 +472,7 @@ describe("agent harness runtime", () => { await executeTool(tools.finalize, { text: "Revenue was $24.9B [[cite:1]] [[cite:2]].", citations: [{ ref: "r1:result:1" }, { ref: "r1:result:2" }], + memoryCitations: [], artifacts: [], unresolved: [], }), @@ -523,6 +545,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -547,6 +570,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages: vi.fn().mockResolvedValue({ analysis: "", @@ -575,6 +599,7 @@ describe("agent harness runtime", () => { const finalize = await executeTool(tools.finalize, { text: "The amount is 5000 yuan [[cite:1]].", citations: [{ ref: "r1:referenced:1" }], + memoryCitations: [], artifacts: [], unresolved: [], }) @@ -593,6 +618,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), recentTurns: [], }) @@ -600,6 +626,7 @@ describe("agent harness runtime", () => { const manifest = { text: "Answer.", citations: [], + memoryCitations: [], artifacts: [], unresolved: [], } @@ -612,6 +639,60 @@ describe("agent harness runtime", () => { expect(state.finalized).toBe(true) }) + it("returns memory refs from memory_search and stores memoryCitations on finalize", async () => { + const search = vi.fn().mockResolvedValue({ + query: "毛利率", + items: [ + { + ref: "mem:1", + itemId: "item_1", + kind: "stance", + abstractL0: "关注毛利率下滑", + overviewL1: "用户把毛利率当作核心观察指标。", + }, + ], + }) + const state: { + finalizedManifest?: OutputManifest + finalized?: boolean + memorySearchInvoked?: boolean + } = {} + const tools = createHarnessTools({ + state, + ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(search), + knowhereTools: makeKnowhereTools(), + recentTurns: [], + }) + + const searchText = await executeTool(tools.memory_search, { + query: "毛利率", + }) + expect(searchText).toContain('') + expect(searchText).toContain('ref="mem:1"') + expect(searchText).toContain('itemId="item_1"') + expect(state.memorySearchInvoked).toBe(true) + expect(search).toHaveBeenCalledWith({ + query: "毛利率", + kinds: undefined, + }) + + const manifest = { + text: "按已有记忆，毛利率是核心观察指标。", + citations: [], + memoryCitations: [ + { ref: "mem:1", itemId: "item_1", kind: "stance" as const }, + ], + artifacts: [], + unresolved: [], + } + expect(await executeTool(tools.finalize, manifest)).toMatchObject({ + ok: true, + memoryCitations: manifest.memoryCitations, + }) + expect(state.finalizedManifest).toEqual(manifest) + }) + it("rejects finalize of cited page images until inspectImage has run", async () => { const ledger = createEvidenceLedger() ledger.addRetrievalResponse(makePageCitationRetrievalResponse()) @@ -623,6 +704,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages: vi.fn().mockResolvedValue({ analysis: "The clause shows 5000 yuan per occurrence.", @@ -640,6 +722,7 @@ describe("agent harness runtime", () => { const manifest = { text: "The contractor pays 5000 yuan per occurrence [[cite:1]].", citations: [{ ref: "r1:referenced:1" }], + memoryCitations: [], artifacts: [], unresolved: [], } @@ -692,6 +775,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages: vi.fn(), recentTurns: [], @@ -700,6 +784,7 @@ describe("agent harness runtime", () => { const result = await executeTool(tools.finalize, { text: "The contractor pays 5000 yuan [[cite:1]].", citations: [{ ref: "r1:result:1" }], + memoryCitations: [], artifacts: [], unresolved: [], }) @@ -725,6 +810,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), recentTurns: [ { @@ -757,6 +843,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), recentTurns: [ { @@ -881,7 +968,7 @@ describe("agent harness runtime", () => { ]) }) - it("keeps normal steps unconstrained before the finalization step", () => { + it("keeps retrieval tools closed until declareIntent allows them", () => { const result = prepareHarnessStep({ stepNumber: 11, messages: [ @@ -892,14 +979,93 @@ describe("agent harness runtime", () => { ], }) - expect(result).toEqual({ - messages: [ - { - role: "user", - content: "Find the penalty amount.", - }, - ], + expect(result.activeTools).toEqual([ + "declareIntent", + "setContextPolicy", + "inspectImage", + "readPriorTurn", + "finalize", + ]) + expect(result.activeTools).not.toContain("memory_search") + expect(result.activeTools).not.toContain("knowhere_search") + }) + + it("opens only memory_search after intent says retrieval may be needed", () => { + const result = prepareHarnessStep({ + stepNumber: 3, + intent: { + task: "answer", + dependsOnPreviousTurn: false, + retrievalNeeded: "maybe", + targetModalities: ["text"], + constraints: {}, + groundingPolicy: "can_use_context", + }, + messages: [], }) + + expect(result.activeTools).toContain("memory_search") + expect(result.activeTools).not.toContain("knowhere_search") + }) + + it("keeps Knowhere tools closed for no_retrieval even after memory_search", () => { + const result = prepareHarnessStep({ + stepNumber: 4, + memorySearchInvoked: true, + intent: { + task: "answer", + dependsOnPreviousTurn: false, + retrievalNeeded: "no", + targetModalities: ["text"], + constraints: {}, + groundingPolicy: "no_retrieval", + }, + messages: [], + }) + + expect(result.activeTools).not.toContain("memory_search") + expect(result.activeTools).not.toContain("knowhere_search") + }) + + it("opens Knowhere tools only after memory_search when sources are required", () => { + const beforeMemory = prepareHarnessStep({ + stepNumber: 3, + intent: { + task: "answer", + dependsOnPreviousTurn: false, + retrievalNeeded: "yes", + targetModalities: ["text"], + constraints: {}, + groundingPolicy: "must_use_sources", + }, + messages: [], + }) + const afterMemory = prepareHarnessStep({ + stepNumber: 4, + memorySearchInvoked: true, + intent: { + task: "answer", + dependsOnPreviousTurn: false, + retrievalNeeded: "yes", + targetModalities: ["text"], + constraints: {}, + groundingPolicy: "must_use_sources", + }, + messages: [], + }) + + expect(beforeMemory.activeTools).toContain("memory_search") + expect(beforeMemory.activeTools).not.toContain("knowhere_search") + expect(afterMemory.activeTools).toEqual( + expect.arrayContaining([ + "memory_search", + "knowhere_search", + "knowhere_list_documents", + "knowhere_get_document_outline", + "knowhere_read_chunks", + "knowhere_grep_chunks", + ]), + ) }) it("forces image inspection before forced finalization when image assets are available", () => { @@ -1034,6 +1200,14 @@ function executeTool(tool: unknown, input: unknown): Promise { return (tool as { execute: (input: unknown) => Promise }).execute(input) } +function makeMemoryTools( + search: MemoryToolRuntime["search"] = vi + .fn() + .mockResolvedValue({ query: "", items: [] }), +): MemoryToolRuntime { + return { search } +} + function makeKnowhereTools( search: KnowhereToolRuntime["search"] = vi.fn(), ): KnowhereToolRuntime { diff --git a/src/agent-harness/runtime.ts b/src/agent-harness/runtime.ts index 073051dd..11a6c3aa 100644 --- a/src/agent-harness/runtime.ts +++ b/src/agent-harness/runtime.ts @@ -10,6 +10,7 @@ import { z } from "zod" import { createEvidenceLedger } from "./ledger" import { getCanonicalImageAssetKey } from "./image-asset-identity" import { knowhereToolText } from "./knowhere-text" +import { memoryToolText } from "./memory-text" import { mergeImageInspectionHighlights } from "./image-highlights" import type { AgentTurn, @@ -27,8 +28,11 @@ import type { IntentFrame, KnowhereSearchTargetContent, KnowhereToolRuntime, + MemorySearchKind, + MemoryToolRuntime, OutputManifest, } from "./types" +import { memorySearchKinds } from "./types" const defaultMaxSteps = 14 const imageInspectionReminderStepNumber = 12 @@ -42,6 +46,7 @@ export type RunAgentHarnessInput = { readonly model: AgentHarnessModel readonly turn: AgentTurnInput readonly knowhereTools: KnowhereToolRuntime + readonly memoryTools: MemoryToolRuntime readonly inspectImages?: InspectImages readonly maxSteps?: number } @@ -55,6 +60,7 @@ type HarnessToolState = { inspectedImageRefs?: string[] imageHighlights?: ImageInspectionHighlights[] toolCalls?: HarnessToolCallTrace[] + memorySearchInvoked?: boolean } type HarnessTools = ReturnType @@ -171,6 +177,17 @@ const outputCitationSchema = z.object({ .optional(), }) +const memoryCitationSchema = z.object({ + ref: z.string().min(1), + itemId: z.string().min(1), + kind: z.enum(memorySearchKinds), +}) + +const memorySearchSchema = z.object({ + query: z.string().min(1), + kinds: z.array(z.enum(memorySearchKinds)).optional(), +}) + const selectedOutputArtifactSchema = z.object({ type: z.enum(["image", "table"]), ref: z.string().min(1), @@ -197,6 +214,7 @@ const outputArtifactSchema = z.union([ const outputManifestSchema = z.object({ text: z.string(), citations: z.array(outputCitationSchema).default([]), + memoryCitations: z.array(memoryCitationSchema).default([]), artifacts: z.array(outputArtifactSchema).default([]), unresolved: z.array(z.string()).default([]), }) @@ -214,6 +232,7 @@ export async function runAgentHarness( state, ledger, knowhereTools: input.knowhereTools, + memoryTools: input.memoryTools, inspectImages: input.inspectImages, recentTurns: input.turn.recentTurns, }) @@ -225,6 +244,8 @@ export async function runAgentHarness( prepareHarnessStep({ messages: stepMessages, stepNumber, + intent: state.intent, + memorySearchInvoked: state.memorySearchInvoked === true, hasUninspectedImageAssets: input.inspectImages !== undefined && hasUninspectedImageAssets({ state, ledger }), @@ -257,10 +278,33 @@ export async function runAgentHarness( } } +const alwaysAvailableTools = [ + "declareIntent", + "setContextPolicy", + "inspectImage", + "readPriorTurn", + "finalize", +] as const + +const fluidRetrievalTools = ["memory_search"] as const + +const crystalRetrievalTools = [ + "knowhere_search", + "knowhere_list_documents", + "knowhere_get_document_outline", + "knowhere_read_chunks", + "knowhere_grep_chunks", +] as const + +/** Reserved third retrieval slot (cognition). Not registered this round. */ +const cognitionRetrievalTools = [] as const + export function prepareHarnessStep(input: { readonly stepNumber: number readonly messages: readonly ModelMessage[] readonly hasUninspectedImageAssets?: boolean + readonly intent?: IntentFrame + readonly memorySearchInvoked?: boolean }): HarnessStepPreparation { const messages = sanitizeHarnessModelMessagesForStep(input.messages) @@ -285,24 +329,59 @@ export function prepareHarnessStep(input: { } } - if (input.stepNumber < forcedFinalizationStepNumber) { - return { messages } + if (input.stepNumber >= forcedFinalizationStepNumber) { + return { + messages: [ + ...messages, + { + role: "user", + content: buildForcedFinalizationFeedback(), + }, + ], + activeTools: ["finalize"], + toolChoice: { + type: "tool", + toolName: "finalize", + }, + } } return { - messages: [ - ...messages, - { - role: "user", - content: buildForcedFinalizationFeedback(), - }, - ], - activeTools: ["finalize"], - toolChoice: { - type: "tool", - toolName: "finalize", - }, + messages, + activeTools: selectHarnessActiveTools({ + intent: input.intent, + memorySearchInvoked: input.memorySearchInvoked === true, + }), + } +} + +function selectHarnessActiveTools(input: { + readonly intent?: IntentFrame + readonly memorySearchInvoked: boolean +}): Array> { + const tools: Array> = [ + ...alwaysAvailableTools, + ] + if (!allowsRetrieval(input.intent)) { + return tools } + + tools.push(...fluidRetrievalTools) + if ( + input.intent?.groundingPolicy === "must_use_sources" && + input.memorySearchInvoked + ) { + tools.push(...crystalRetrievalTools) + } + tools.push(...cognitionRetrievalTools) + return tools +} + +function allowsRetrieval(intent?: IntentFrame): boolean { + if (!intent) return false + return ( + intent.groundingPolicy !== "no_retrieval" && intent.retrievalNeeded !== "no" + ) } export function sanitizeHarnessModelMessagesForStep( @@ -410,6 +489,7 @@ export function createHarnessTools(input: { readonly state: HarnessToolState readonly ledger: ReturnType readonly knowhereTools: KnowhereToolRuntime + readonly memoryTools: MemoryToolRuntime readonly inspectImages?: InspectImages readonly recentTurns: readonly AgentTurn[] }) { @@ -446,6 +526,26 @@ export function createHarnessTools(input: { }), }), + memory_search: tool({ + description: + "Search distilled fluid memory for this workspace. Returns tagged text with memory refs such as mem:1. Use this before Knowhere document search.", + inputSchema: memorySearchSchema, + execute: async (request) => + traceToolCall(input.state, { + toolName: "memory_search", + inputSummary: summarizeMemorySearchRequest(request), + execute: async () => { + const output = await executeMemorySearch({ + memoryTools: input.memoryTools, + request, + }) + input.state.memorySearchInvoked = true + return output + }, + summarizeOutput: summarizeMemoryTextOutput, + }), + }), + knowhere_search: tool({ description: "Search Knowhere for relevant Notebook evidence. Returns tagged text with evidence refs such as r1:result:1 and asset refs such as asset:r1:result:1.", @@ -628,6 +728,7 @@ export function createHarnessTools(input: { "Finalize the user-facing output manifest. This is the only final answer " + "contract. Artifacts listed here with display=true are the exact set of " + "images/tables shown to the user; cite evidence refs when available. " + + "Use citations for Knowhere evidence and memoryCitations for fluid memory refs. " + "Cited page/image assets must be inspected with inspectImage first.", inputSchema: outputManifestSchema, execute: async (manifest) => @@ -912,6 +1013,7 @@ type KnowhereToolOperation = | "read_chunks" | "grep_chunks" +type MemorySearchToolRequest = z.infer type KnowhereSearchToolRequest = z.infer type KnowhereDocumentReferenceRequest = z.infer< typeof knowhereDocumentReferenceSchema @@ -925,6 +1027,24 @@ type DocumentReferenceSummary = { readonly hasRevisionKey: boolean } +async function executeMemorySearch(input: { + readonly memoryTools: MemoryToolRuntime + readonly request: MemorySearchToolRequest +}): Promise { + try { + const response = await input.memoryTools.search({ + query: input.request.query, + kinds: input.request.kinds, + }) + return memoryToolText.formatSearch(response) + } catch (error) { + return memoryToolText.formatError({ + operation: "search", + message: formatUnknownError(error), + }) + } +} + async function executeKnowhereSearch(input: { readonly ledger: ReturnType readonly knowhereTools: KnowhereToolRuntime @@ -1114,6 +1234,25 @@ function summarizeContextPolicy(policy: ContextPolicy): unknown { } } +function summarizeMemorySearchRequest(request: { + readonly query: string + readonly kinds?: readonly MemorySearchKind[] +}): unknown { + return { + query: request.query, + kinds: request.kinds, + } +} + +function summarizeMemoryTextOutput(output: unknown): unknown { + if (typeof output !== "string") return output + return { + ok: !output.includes('status="error"'), + textLength: output.length, + itemCount: countOccurrences(output, " artifact.display) .length, @@ -1255,6 +1395,9 @@ function summarizeFinalizeOutput(output: unknown): unknown { ok: output.ok, textLength: typeof output.text === "string" ? output.text.length : 0, citationCount: Array.isArray(output.citations) ? output.citations.length : 0, + memoryCitationCount: Array.isArray(output.memoryCitations) + ? output.memoryCitations.length + : 0, artifactCount: Array.isArray(output.artifacts) ? output.artifacts.length : 0, unresolvedCount: Array.isArray(output.unresolved) ? output.unresolved.length @@ -1282,12 +1425,17 @@ export function buildHarnessSystemPrompt(turn: AgentTurnInput): string { "1. Call declareIntent when it helps you plan the response. Capture constraints like a requested image/table count in constraints.desiredCount.", "2. Call setContextPolicy when prior turns may influence this turn.", "3. When the policy needs prior-turn detail (references or corrections), call readPriorTurn for the relevant ids.", - "4. Call knowhere_search when relevance search is needed. Use knowhere_list_documents, knowhere_get_document_outline, knowhere_read_chunks, and knowhere_grep_chunks for focused document reads.", + "4. Call memory_search first when known fluid memory may answer the request. Call knowhere_search only when memory is insufficient and groundingPolicy requires citing source documents. Use knowhere_list_documents, knowhere_get_document_outline, knowhere_read_chunks, and knowhere_grep_chunks for focused document reads.", "5. After Knowhere returns image/page asset refs, call inspectImage on the page/image assets you will cite before finalize. This supplies OCR/visual context and provenance boxes.", "6. Inspect each unique cited page once; retrieval already bounds the available evidence set.", "7. knowhere_read_chunks returns complete chunk bodies; control size with page/pageSize, sectionPath, startChunk/endChunk, chunkId, and chunkType.", "8. Call finalize with text, citations, artifacts, and unresolved issues when you are ready to answer.", "", + "Retrieval rules:", + "- First use memory_search to see whether known fluid memory can answer directly.", + "- Call knowhere_search only when memory is insufficient and groundingPolicy requires citing source documents.", + "- Do not treat every question as a document-retrieval task.", + "", "Context rules:", "- If the current user request is unrelated to prior turns, set carryHistory to none and do not reuse prior topics.", "- If the user corrects a previous answer, set carryHistory to repair_previous, read the relevant prior turn, then re-retrieve and re-answer using the correction.", @@ -1347,6 +1495,7 @@ function buildFallbackManifest(text: string): OutputManifest { return { text, citations: [], + memoryCitations: [], artifacts: [], unresolved: text ? [] : ["The agent did not finalize an output manifest."], } diff --git a/src/agent-harness/types.ts b/src/agent-harness/types.ts index edd9c2f6..82f38ab5 100644 --- a/src/agent-harness/types.ts +++ b/src/agent-harness/types.ts @@ -131,6 +131,43 @@ export type KnowhereToolRuntime = { ) => Promise } +export const memorySearchKinds = [ + "indicator_pref", + "stance", + "decision_rule", + "entity_of_interest", +] as const + +export type MemorySearchKind = (typeof memorySearchKinds)[number] + +export type MemorySearchRequest = { + readonly query: string + readonly kinds?: readonly MemorySearchKind[] +} + +export type MemorySearchItem = { + readonly ref: string + readonly itemId: string + readonly kind: MemorySearchKind + readonly abstractL0: string + readonly overviewL1: string +} + +export type MemorySearchResponse = { + readonly query: string + readonly items: readonly MemorySearchItem[] +} + +export type MemoryToolRuntime = { + readonly search: (input: MemorySearchRequest) => Promise +} + +export type MemoryCitation = { + readonly ref: string + readonly itemId: string + readonly kind: MemorySearchKind +} + export type EvidenceChunk = { readonly ref: string readonly kind: "result" | "referenced_chunk" | "read_chunk" | "grep_match" @@ -254,6 +291,7 @@ export type OutputArtifactView = OutputArtifact | DerivedTableArtifact export type OutputManifest = { readonly text: string readonly citations: readonly OutputCitation[] + readonly memoryCitations: readonly MemoryCitation[] readonly artifacts: readonly OutputArtifactView[] readonly unresolved: readonly string[] } diff --git a/src/domains/chat/citations.test.ts b/src/domains/chat/citations.test.ts index b381a83a..1335de3c 100644 --- a/src/domains/chat/citations.test.ts +++ b/src/domains/chat/citations.test.ts @@ -64,6 +64,21 @@ describe("toChatCitationViews", () => { expect(citations[0]?.pageCitationPageNumber).toBe(26) }) + it("copies the chunk id onto the citation when the retrieval result has one", () => { + const citations = toChatCitationViews( + [makeRetrievalResult({ chunkId: "chunk_123" })], + "Grounded answer.", + ) + + expect(citations[0]?.chunkId).toBe("chunk_123") + }) + + it("omits chunkId when the retrieval result has none", () => { + const citations = toChatCitationViews([makeRetrievalResult()], "Grounded answer.") + + expect(citations[0]).not.toHaveProperty("chunkId") + }) + it("copies inspect-image provenance boxes onto the citation", () => { const citations = toChatCitationViews( [ diff --git a/src/domains/chat/citations.ts b/src/domains/chat/citations.ts index 9fb6c8dc..18304aa4 100644 --- a/src/domains/chat/citations.ts +++ b/src/domains/chat/citations.ts @@ -20,6 +20,7 @@ export function toChatCitationViews( content: result.content, chunkType: result.chunkType, score: result.score, + ...(result.chunkId ? { chunkId: result.chunkId } : {}), ...(result.assetUrl ? { assetUrl: result.assetUrl } : {}), ...(result.pageCitationAssetUrl ? { pageCitationAssetUrl: result.pageCitationAssetUrl } diff --git a/src/domains/chat/commit-turn.test.ts b/src/domains/chat/commit-turn.test.ts new file mode 100644 index 00000000..62e20b31 --- /dev/null +++ b/src/domains/chat/commit-turn.test.ts @@ -0,0 +1,138 @@ +import { Either } from "effect" +import { beforeEach, describe, expect, it, vi } from "vitest" + +const mocks = vi.hoisted(() => ({ + handleChatTurn: vi.fn(), + triggerMemoryExtraction: vi.fn(), + recordActivations: vi.fn(), +})) + +vi.mock("./service", () => ({ + handleChatTurn: mocks.handleChatTurn, +})) + +vi.mock("@/domains/memory/extract-trigger", () => ({ + triggerMemoryExtraction: mocks.triggerMemoryExtraction, +})) + +vi.mock("@/domains/retrieval-activation/service", () => ({ + retrievalActivationService: { + recordActivations: mocks.recordActivations, + }, +})) + +import { commitChatTurn } from "./commit-turn" +import type { Workspace } from "@/infrastructure/db/schema" + +describe("commitChatTurn", () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.recordActivations.mockResolvedValue(1) + }) + + it("records fluid memory activations from finalize after a successful turn", async () => { + mocks.handleChatTurn.mockImplementation(async (input) => { + await input.generateAnswer({ + question: "毛利率", + messages: [], + sources: [], + excludedSourceIds: [], + searchSources: vi.fn(), + }) + return Either.right({ + threadId: "thread_1", + messages: [ + { id: "msg_user", role: "user", content: "毛利率" }, + { + id: "msg_assistant", + role: "assistant", + content: "按已有记忆。", + citations: [ + { + chunkId: "chunk_1", + chunkType: "text", + score: 0.9, + source: { documentId: "doc_1" }, + }, + ], + }, + ], + }) + }) + + const generateAnswer = vi.fn(async () => ({ + manifest: { + text: "按已有记忆。", + citations: [], + memoryCitations: [ + { ref: "mem:1", itemId: "item_1", kind: "stance" as const }, + ], + artifacts: [], + unresolved: [], + }, + trace: { + ledger: { + retrievalCount: 0, + chunks: [], + assets: [], + evidenceText: [], + stopReasons: [], + failureReasons: [], + decisionTraces: [], + }, + finalized: true, + priorTurnReads: [], + toolCalls: [], + imageHighlights: [], + validationErrors: [], + revisionsUsed: 0, + }, + })) + + const result = await commitChatTurn({ + workspace: makeWorkspace(), + sources: [], + question: "毛利率", + excludedSourceIds: [], + retrieval: { query: vi.fn() }, + generateAnswer, + repository: { + ensureDefaultChatThread: vi.fn(), + findChatThreadInWorkspace: vi.fn(), + listMessagesForThread: vi.fn(), + appendMessageToThread: vi.fn(), + }, + }) + + expect(Either.isRight(result)).toBe(true) + expect(mocks.triggerMemoryExtraction).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + threadId: "thread_1", + userMessageId: "msg_user", + assistantMessageId: "msg_assistant", + }) + expect(mocks.recordActivations).toHaveBeenCalledWith([ + { + workspaceId: "workspace_1", + unitType: "crystal_chunk", + unitRef: "doc_1:chunk_1", + }, + ]) + expect(mocks.recordActivations).toHaveBeenCalledWith([ + { + workspaceId: "workspace_1", + unitType: "fluid_memory", + unitRef: "item_1", + }, + ]) + }) +}) + +function makeWorkspace(): Workspace { + return { + id: "workspace_1", + userId: "user_1", + namespace: "notebook-namespace", + createdAt: new Date("2026-09-10T00:00:00Z"), + } +} diff --git a/src/domains/chat/commit-turn.ts b/src/domains/chat/commit-turn.ts new file mode 100644 index 00000000..373bb3a6 --- /dev/null +++ b/src/domains/chat/commit-turn.ts @@ -0,0 +1,131 @@ +import { Either } from "effect" + +import type { MemoryCitation } from "@/agent-harness" +import { generateAgenticOutputManifest } from "./prompt" +import { triggerMemoryExtraction } from "@/domains/memory/extract-trigger" +import { retrievalActivationService } from "@/domains/retrieval-activation/service" +import { toChunkUnitRef } from "@/domains/retrieval-activation/types" +import { summarizeUnknownError } from "@/lib/format-log-value" +import { logger } from "@/lib/logger" +import type { ChatCitationView } from "./types" +import { + handleChatTurn, + type ChatTurnError, + type ChatTurnValue, +} from "./service" + +type CommitChatTurnInput = Parameters[0] + +/** + * Production chat-turn commit used by the HTTP route and the 观心 batch + * script: answer → persist → extract fluid memory → record cited + * crystal/memory activations. + */ +export async function commitChatTurn( + input: CommitChatTurnInput, +): Promise> { + let memoryCitations: readonly MemoryCitation[] = [] + const result = await handleChatTurn({ + ...input, + generateAnswer: async (generateInput) => { + const generated = await input.generateAnswer({ + ...generateInput, + }) + memoryCitations = generated.manifest.memoryCitations + return generated + }, + }) + + if (Either.isRight(result)) { + void triggerMemoryExtraction({ + workspaceId: input.workspace.id, + threadId: result.right.threadId, + userMessageId: result.right.messages[0].id, + assistantMessageId: result.right.messages[1].id, + }) + void recordChunkActivations({ + workspaceId: input.workspace.id, + citations: result.right.messages[1].citations, + }) + void recordMemoryActivations({ + workspaceId: input.workspace.id, + memoryCitations, + }) + } + + return result +} + +export async function commitAgenticChatTurn( + input: Omit, +): Promise> { + return commitChatTurn({ + ...input, + generateAnswer: (generateInput) => + generateAgenticOutputManifest({ + ...generateInput, + workspaceId: input.workspace.id, + }), + }) +} + +/** + * Fire-and-forget activation ledger write for crystal chunks actually cited + * in this turn's answer. Only citations with both a documentId and a + * chunkId count — a citation missing either can't be identified down to a + * chunk (see ChatCitationView / RetrievalResultView). + */ +export async function recordChunkActivations(input: { + readonly workspaceId: string + readonly citations: readonly ChatCitationView[] | undefined +}): Promise { + const activationInputs = (input.citations ?? []).flatMap((citation) => { + const documentId = citation.source.documentId + const chunkId = citation.chunkId + if (!documentId || !chunkId) return [] + return [ + { + workspaceId: input.workspaceId, + unitType: "crystal_chunk" as const, + unitRef: toChunkUnitRef({ documentId, chunkId }), + }, + ] + }) + if (activationInputs.length === 0) return + + try { + await retrievalActivationService.recordActivations(activationInputs) + } catch (error) { + logger.warn("chat: failed to record chunk activations", { + workspaceId: input.workspaceId, + chunkCount: activationInputs.length, + error: summarizeUnknownError(error), + }) + } +} + +/** + * Fire-and-forget activation ledger write for fluid memory items actually + * cited in this turn's finalize output. + */ +export async function recordMemoryActivations(input: { + readonly workspaceId: string + readonly memoryCitations: readonly MemoryCitation[] +}): Promise { + const activationInputs = input.memoryCitations.map((citation) => ({ + workspaceId: input.workspaceId, + unitType: "fluid_memory" as const, + unitRef: citation.itemId, + })) + if (activationInputs.length === 0) return + + try { + await retrievalActivationService.recordActivations(activationInputs) + } catch (error) { + logger.warn("chat: failed to record memory activations", { + workspaceId: input.workspaceId, + memoryCount: activationInputs.length, + error: summarizeUnknownError(error), + }) + } +} diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index 1bc9b606..021ebb22 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -1040,6 +1040,7 @@ describe("answerQuestionWithRetrieval", () => { manifest: { text: `Use this image. ${rawAssetUrl}`, citations: [], + memoryCitations: [], artifacts: [ { type: "image", @@ -1773,6 +1774,7 @@ describe("answerQuestionWithRetrieval", () => { manifest: { text: "已找到相关身份证图片，见下方图片。", citations: [], + memoryCitations: [], artifacts: [ { type: "image", @@ -2063,6 +2065,7 @@ describe("answerQuestionWithRetrieval", () => { }, }, ], + memoryCitations: [], artifacts: [], unresolved: [], }); @@ -2081,7 +2084,11 @@ describe("answerQuestionWithRetrieval", () => { sources: [makeSource()], excludedSourceIds: [], retrieval, - generateAnswer: generateAgenticOutputManifest, + generateAnswer: (input) => + generateAgenticOutputManifest({ + ...input, + workspaceId: "workspace_1", + }), messages: [], }), ); @@ -2128,6 +2135,7 @@ describe("answerQuestionWithRetrieval", () => { manifest: { text: "", citations: [], + memoryCitations: [], artifacts: [ { type: "image", @@ -2243,6 +2251,7 @@ describe("answerQuestionWithRetrieval", () => { manifest: { text: "I organized the comparison into a table.", citations: [], + memoryCitations: [], artifacts: [ { type: "derived_table", @@ -2639,6 +2648,7 @@ describe("answerQuestionWithRetrieval", () => { content: "", chunkType: "image", score: null, + chunkId: "chunk_1", assetUrl: "https://blob.example/images/launch.jpg", source: { documentId: "doc_spacex", @@ -2699,6 +2709,7 @@ describe("generateAgenticOutputManifest", () => { }, }, ], + memoryCitations: [], artifacts: [ { type: "image", @@ -2739,6 +2750,7 @@ describe("generateAgenticOutputManifest", () => { }); const result = await generateAgenticOutputManifest({ + workspaceId: "workspace_1", question: "请只返回冯荣洲的 2 张身份证图片", messages: [ { @@ -2836,6 +2848,7 @@ describe("generateAgenticOutputManifest", () => { }, }, ], + memoryCitations: [], artifacts: [ { type: "image", @@ -2885,6 +2898,7 @@ describe("generateAgenticOutputManifest", () => { }); const result = await generateAgenticOutputManifest({ + workspaceId: "workspace_1", question: "Inspect and show the ID card image.", messages: [], sources: [ @@ -2976,6 +2990,7 @@ describe("generateAgenticOutputManifest", () => { }, }, ], + memoryCitations: [], artifacts: [], unresolved: [], }); @@ -3029,6 +3044,7 @@ describe("generateAgenticOutputManifest", () => { }); const result = await generateAgenticOutputManifest({ + workspaceId: "workspace_1", question: "承包人自行修改发包人审批的进度时需要赔偿多少违约金？", messages: [], sources: [ @@ -3126,6 +3142,7 @@ describe("generateAgenticOutputManifest", () => { }, }, ], + memoryCitations: [], artifacts: [1, 2, 3].map((index) => ({ type: "image", ref: `asset:r1:result:${index}`, @@ -3148,6 +3165,7 @@ describe("generateAgenticOutputManifest", () => { }, }, ], + memoryCitations: [], artifacts: [1, 2].map((index) => ({ type: "image", ref: `asset:r1:result:${index}`, @@ -3188,6 +3206,7 @@ describe("generateAgenticOutputManifest", () => { }); const result = await generateAgenticOutputManifest({ + workspaceId: "workspace_1", question: "只要 2 张身份证图片", messages: [], sources: [ @@ -3322,6 +3341,7 @@ function makeHarnessRunResult(text: string): HarnessRunResult { manifest: { text, citations: [], + memoryCitations: [], artifacts: [], unresolved: [], }, @@ -3371,6 +3391,7 @@ function makeHarnessRunResultWithLedger( manifest: { text, citations: input.citations ?? [], + memoryCitations: [], artifacts: input.artifacts ?? [], unresolved: [], }, diff --git a/src/domains/chat/memory-tools.test.ts b/src/domains/chat/memory-tools.test.ts new file mode 100644 index 00000000..ad083a90 --- /dev/null +++ b/src/domains/chat/memory-tools.test.ts @@ -0,0 +1,104 @@ +import { beforeEach, describe, expect, it, vi } from "vitest" + +import { DISTILL_DEDUP_CANDIDATES_PER_KIND } from "@/domains/memory/distill-config" +import type { FluidMemoryItem } from "@/infrastructure/db/schema" + +const findDedupCandidates = vi.fn() + +vi.mock("@/domains/memory/service", () => ({ + memoryService: { + findDedupCandidates: (...args: unknown[]) => findDedupCandidates(...args), + }, +})) + +describe("notebookMemoryTools", () => { + beforeEach(() => { + findDedupCandidates.mockReset() + }) + + it("queries all four kinds and assigns mem refs in kind order", async () => { + findDedupCandidates.mockImplementation( + async (_workspaceId: string, kind: string) => { + if (kind === "stance") return [makeMemoryItem({ id: "item_stance" })] + if (kind === "entity_of_interest") { + return [makeMemoryItem({ id: "item_entity", kind: "entity_of_interest" })] + } + return [] + }, + ) + const { notebookMemoryTools } = await import("./memory-tools") + const runtime = notebookMemoryTools.createRuntime({ + workspaceId: "workspace_1", + }) + + const response = await runtime.search({ query: "毛利率 英伟达" }) + + expect(findDedupCandidates).toHaveBeenCalledTimes(4) + expect(findDedupCandidates.mock.calls.map((call) => call[1])).toEqual([ + "indicator_pref", + "stance", + "decision_rule", + "entity_of_interest", + ]) + expect(findDedupCandidates.mock.calls[0]?.[3]).toBe( + DISTILL_DEDUP_CANDIDATES_PER_KIND, + ) + expect(response).toEqual({ + query: "毛利率 英伟达", + items: [ + expect.objectContaining({ + ref: "mem:1", + itemId: "item_stance", + kind: "stance", + }), + expect.objectContaining({ + ref: "mem:2", + itemId: "item_entity", + kind: "entity_of_interest", + }), + ], + }) + }) + + it("searches only the requested kinds", async () => { + findDedupCandidates.mockResolvedValue([]) + const { notebookMemoryTools } = await import("./memory-tools") + const runtime = notebookMemoryTools.createRuntime({ + workspaceId: "workspace_1", + }) + + await runtime.search({ + query: "PE", + kinds: ["indicator_pref"], + }) + + expect(findDedupCandidates).toHaveBeenCalledTimes(1) + expect(findDedupCandidates).toHaveBeenCalledWith( + "workspace_1", + "indicator_pref", + expect.any(Array), + DISTILL_DEDUP_CANDIDATES_PER_KIND, + ) + }) +}) + +function makeMemoryItem( + overrides: Partial = {}, +): FluidMemoryItem { + return { + id: "item_1", + workspaceId: "workspace_1", + kind: "stance", + payload: { statement: "s", scope: "scope", rationale: "r" }, + abstractL0: "abstract", + overviewL1: "overview", + sourceMessageId: null, + confidence: 0.8, + status: "active", + deactivationReason: null, + version: 1, + createdAt: new Date("2026-09-10T00:00:00Z"), + updatedAt: new Date("2026-09-10T00:00:00Z"), + ...overrides, + } +} diff --git a/src/domains/chat/memory-tools.ts b/src/domains/chat/memory-tools.ts new file mode 100644 index 00000000..63d48b11 --- /dev/null +++ b/src/domains/chat/memory-tools.ts @@ -0,0 +1,66 @@ +import type { + MemorySearchItem, + MemorySearchRequest, + MemorySearchResponse, + MemoryToolRuntime, +} from "@/agent-harness" +import { DISTILL_DEDUP_CANDIDATES_PER_KIND } from "@/domains/memory/distill-config" +import { tokenizeMemoryText } from "@/domains/memory/search-index" +import { memoryService } from "@/domains/memory/service" +import { fluidMemoryKinds, isFluidMemoryKind } from "@/domains/memory/types" +import type { FluidMemoryItem } from "@/infrastructure/db/schema" + +type NotebookMemoryToolsInput = { + readonly workspaceId: string +} + +export const notebookMemoryTools = { + createRuntime(input: NotebookMemoryToolsInput): MemoryToolRuntime { + return { + search: (request) => searchWorkspaceMemory(input.workspaceId, request), + } + }, +} as const + +async function searchWorkspaceMemory( + workspaceId: string, + request: MemorySearchRequest, +): Promise { + const tokens = tokenizeMemoryText(request.query).map((entry) => entry.token) + const kinds = request.kinds ?? fluidMemoryKinds + const items: MemorySearchItem[] = [] + + for (const kind of kinds) { + const candidates = await memoryService.findDedupCandidates( + workspaceId, + kind, + tokens, + // Same per-kind cap as the existing findDedupCandidates caller. + DISTILL_DEDUP_CANDIDATES_PER_KIND, + ) + for (const candidate of candidates) { + const item = toMemorySearchItem(candidate, items.length + 1) + if (item) items.push(item) + } + } + + return { + query: request.query, + items, + } +} + +function toMemorySearchItem( + item: FluidMemoryItem, + index: number, +): MemorySearchItem | null { + if (!isFluidMemoryKind(item.kind)) return null + + return { + ref: `mem:${index}`, + itemId: item.id, + kind: item.kind, + abstractL0: item.abstractL0, + overviewL1: item.overviewL1, + } +} diff --git a/src/domains/chat/prompt.ts b/src/domains/chat/prompt.ts index a4ae326d..d61177ea 100644 --- a/src/domains/chat/prompt.ts +++ b/src/domains/chat/prompt.ts @@ -17,12 +17,14 @@ import type { SearchSources, } from "./contracts" import { notebookKnowhereTools } from "./knowhere-tools" +import { notebookMemoryTools } from "./memory-tools" const RECENT_CONTEXT_MESSAGE_LIMIT = 8 const CONTEXT_CONTENT_CHAR_LIMIT = 900 const SOURCE_CONTEXT_LIMIT = 12 type GenerateAgenticOutputManifestInput = { + workspaceId: string question: string messages: readonly ChatHistoryMessage[] sources: readonly Source[] @@ -63,6 +65,9 @@ export const generateAgenticOutputManifestEffect = ( notebookKnowhereTools.createSearchOnlyRuntime({ searchSources: input.searchSources, }), + memoryTools: notebookMemoryTools.createRuntime({ + workspaceId: input.workspaceId, + }), ...(input.inspectImages ? { inspectImages: input.inspectImages } : {}), }), ) diff --git a/src/domains/chat/route-answer.ts b/src/domains/chat/route-answer.ts index cc579b05..0a7cbb4d 100644 --- a/src/domains/chat/route-answer.ts +++ b/src/domains/chat/route-answer.ts @@ -1,9 +1,6 @@ import { Cause, Effect, Either, Option } from "effect" -import { - generateAgenticOutputManifest, - parseChatRequestBody, -} from "@/domains/chat" +import { parseChatRequestBody } from "@/domains/chat" import type { ImageInspectionAsset, ImageInspectionRequest, @@ -11,16 +8,15 @@ import type { ImageInspectionSkippedAsset, InspectImages, } from "@/agent-harness" +import { commitAgenticChatTurn } from "@/domains/chat/commit-turn" import { normalizeImageInspectionHighlights } from "@/agent-harness/image-highlights" import { generateImageInspectionModelResult } from "@/domains/chat/image-inspection-model" import { hardenChatMediaAssetUrls } from "@/domains/chat/media-asset-hardening" import { - handleChatTurn, type ChatTurnError, type ChatTurnValue, } from "@/domains/chat/service" import { chatTurnPersistence } from "@/domains/chat/chat-turn-persistence" -import { triggerMemoryExtraction } from "@/domains/memory/extract-trigger" import { startBackgroundReconciliation } from "@/domains/sources/background-reconcile" import { BlobParsedDocumentStorage } from "@/domains/sources/parsed-document-blob-storage" import { sourceWorkflowRuntime } from "@/domains/sources/workflow-runtime" @@ -140,7 +136,7 @@ const answerChatEffect = (input: AnswerChatInput) => const result: Either.Either = yield* Effect.tryPromise(() => - handleChatTurn({ + commitAgenticChatTurn({ workspace, sources, question: body.value.question, @@ -150,7 +146,6 @@ const answerChatEffect = (input: AnswerChatInput) => retrieval: client.retrieval, knowledge: knowhereResources.knowledge, remoteDocumentClient: client, - generateAnswer: generateAgenticOutputManifest, hardenChatAssetUrl, hardenMediaAssetUrls: ({ results, artifacts }) => hardenChatMediaAssetUrls({ @@ -197,17 +192,8 @@ const answerChatEffect = (input: AnswerChatInput) => return Either.match(result, { onLeft: (error): RouteResponse => routeResult.error(error.status, error.message), - onRight: (value): RouteResponse => { - // Fire-and-forget: extract fluid memory from this turn without - // blocking the chat response. - void triggerMemoryExtraction({ - workspaceId: workspace.id, - threadId: value.threadId, - userMessageId: value.messages[0].id, - assistantMessageId: value.messages[1].id, - }) - return routeResult.ok(value) - }, + onRight: (value): RouteResponse => + routeResult.ok(value), }) }) diff --git a/src/domains/chat/route-service.test.ts b/src/domains/chat/route-service.test.ts index 02eceb9b..54e8dae0 100644 --- a/src/domains/chat/route-service.test.ts +++ b/src/domains/chat/route-service.test.ts @@ -191,7 +191,7 @@ describe("chat route services", () => { useAgentic: true, excludedSourceIds: ["source_skipped"], retrieval: client.retrieval, - generateAnswer: mocks.generateAgenticOutputManifest, + generateAnswer: expect.any(Function), hardenChatAssetUrl: expect.any(Function), repository: expect.objectContaining({ appendMessageToThread: expect.any(Function), diff --git a/src/domains/chat/service.test.ts b/src/domains/chat/service.test.ts index b0652519..0ac23870 100644 --- a/src/domains/chat/service.test.ts +++ b/src/domains/chat/service.test.ts @@ -83,6 +83,39 @@ describe("handleChatTurn", () => { }); }); + it("allows a turn with no local sources so remote retrieval can still run", async () => { + const retrieval = { + query: vi.fn().mockResolvedValue({ + results: [makeRetrievalResult()], + evidenceText: "Grounding content", + referencedChunks: [], + namespace: "notebook-namespace", + query: "What does the document say?", + routerUsed: "workflow_single_step", + answerText: null, + }), + }; + const repository = makeRepository(); + const generateAnswer = vi.fn(async ({ searchSources }) => { + await searchSources({ query: "What does the document say?" }); + return makeHarnessRunResult("Grounded answer."); + }); + + const result = await handleChatTurn({ + workspace: makeWorkspace(), + sources: [], + question: "What does the document say?", + excludedSourceIds: [], + retrieval, + generateAnswer, + repository, + }); + + expect(Either.isRight(result)).toBe(true); + expect(generateAnswer).toHaveBeenCalled(); + expect(repository.appendMessageToThread).toHaveBeenCalled(); + }); + it("rejects chat before any source is ready without calling retrieval", async () => { const retrieval = { query: vi.fn() }; const repository = makeRepository(); @@ -334,6 +367,7 @@ function makeHarnessRunResult(text: string): HarnessRunResult { manifest: { text, citations: [], + memoryCitations: [], artifacts: [], unresolved: [], }, diff --git a/src/domains/chat/service.ts b/src/domains/chat/service.ts index 23bfa68d..b2c5bce2 100644 --- a/src/domains/chat/service.ts +++ b/src/domains/chat/service.ts @@ -85,7 +85,7 @@ export const handleChatTurnEffect = (input: ChatTurnInput) => const readySources = input.sources.filter( (source) => source.status === "ready" && source.knowhereDocumentId, ) - if (readySources.length === 0) { + if (input.sources.length > 0 && readySources.length === 0) { return yield* Effect.fail(noReadySources) } diff --git a/src/domains/chat/types.ts b/src/domains/chat/types.ts index 1d6f88f2..e10644f6 100644 --- a/src/domains/chat/types.ts +++ b/src/domains/chat/types.ts @@ -5,6 +5,7 @@ export type RetrievalResultView = { readonly content: string readonly chunkType: string readonly score: number | null + readonly chunkId?: string readonly assetUrl?: string readonly pageCitationAssetUrl?: string readonly pageCitationPageNumber?: number diff --git a/src/domains/memory/decay-candidates.test.ts b/src/domains/memory/decay-candidates.test.ts new file mode 100644 index 00000000..7635c1fb --- /dev/null +++ b/src/domains/memory/decay-candidates.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest" + +import { selectDecayCandidates } from "./decay-candidates" + +const NOW = new Date("2026-01-08T00:00:00Z") +const A_YEAR_AGO = new Date("2025-01-08T00:00:00Z") + +describe("selectDecayCandidates", () => { + it("flags an old, never-activated item below the threshold", () => { + const candidates = selectDecayCandidates({ + items: [{ id: "item_1", kind: "stance", createdAt: A_YEAR_AGO }], + activationsById: new Map(), + now: NOW, + scoreThreshold: 0.3, + }) + + expect(candidates).toEqual([ + { id: "item_1", kind: "stance", score: expect.any(Number), activationCount: 0 }, + ]) + expect(candidates[0]!.score).toBeLessThan(0.3) + }) + + it("does not flag a freshly created item even with no activations", () => { + const candidates = selectDecayCandidates({ + items: [{ id: "item_1", kind: "stance", createdAt: NOW }], + activationsById: new Map(), + now: NOW, + scoreThreshold: 0.3, + }) + + expect(candidates).toEqual([]) + }) + + it("does not flag an old item that was recently activated", () => { + const candidates = selectDecayCandidates({ + items: [{ id: "item_1", kind: "stance", createdAt: A_YEAR_AGO }], + activationsById: new Map([ + ["item_1", { activationCount: 3, lastActivatedAt: NOW }], + ]), + now: NOW, + scoreThreshold: 0.3, + }) + + expect(candidates).toEqual([]) + }) + + it("only flags items strictly below the given threshold", () => { + const items = [ + { id: "item_1", kind: "stance" as const, createdAt: A_YEAR_AGO }, + { id: "item_2", kind: "stance" as const, createdAt: NOW }, + ] + + const noneFlagged = selectDecayCandidates({ + items, + activationsById: new Map(), + now: NOW, + scoreThreshold: 0, + }) + expect(noneFlagged).toEqual([]) + + const allFlagged = selectDecayCandidates({ + items, + activationsById: new Map(), + now: NOW, + scoreThreshold: 1, + }) + expect(allFlagged.map((c) => c.id).sort()).toEqual(["item_1", "item_2"]) + }) +}) diff --git a/src/domains/memory/decay-candidates.ts b/src/domains/memory/decay-candidates.ts new file mode 100644 index 00000000..159176e6 --- /dev/null +++ b/src/domains/memory/decay-candidates.ts @@ -0,0 +1,64 @@ +import { computeDecayScore } from "@/domains/retrieval-activation/decay-score" + +/** + * Confirmed decay-candidate threshold: at `BASE_HALF_LIFE_DAYS` (14), a + * never-activated item (ceiling 0.5) crosses this after ~24.3 days of + * silence; one activated once after ~60 days; one activated 3x after ~135 + * days. Chosen by simulating `computeDecayScore`'s actual day-counts across + * activation counts and confirming the resulting grace periods, not picked + * a priori — see the `记忆衰减聚类收尾方案` plan. + */ +export const DEFAULT_DECAY_SCORE_THRESHOLD = 0.15 + +export type DecayableItem = { + readonly id: string + /** Raw `fluid_memory_items.kind` column value — carried through, not validated. */ + readonly kind: string + readonly createdAt: Date +} + +export type ItemActivationStats = { + readonly activationCount: number + readonly lastActivatedAt: Date | null +} + +export type DecayCandidate = { + readonly id: string + readonly kind: string + readonly score: number + readonly activationCount: number +} + +/** + * Pure selection: given active fluid_memory items and their (possibly + * absent) activation ledger rows, return the ones whose decay score is + * below `scoreThreshold`. Does not decide the threshold itself and does not + * write anything — per the plan, a decay score crossing the line only + * produces a *candidate*; moving it to `inactive` is a separate, explicit + * step (see `memoryRepository.deactivateDecayedItemsEffect`). + * + * The anchor for an item with no ledger row (never activated) is its own + * `createdAt` — a real, meaningful signal here (unlike a crystal chunk, + * where "no row" means "no signal at all"), so a never-activated item still + * decays normally from the moment it was created. + */ +export function selectDecayCandidates(input: { + readonly items: readonly DecayableItem[] + readonly activationsById: ReadonlyMap + readonly now: Date + readonly scoreThreshold: number +}): readonly DecayCandidate[] { + return input.items.flatMap((item) => { + const activation = input.activationsById.get(item.id) + const activationCount = activation?.activationCount ?? 0 + const anchorAt = activation?.lastActivatedAt ?? item.createdAt + const score = computeDecayScore({ + activationCount, + anchorAt, + now: input.now, + }) + return score < input.scoreThreshold + ? [{ id: item.id, kind: item.kind, score, activationCount }] + : [] + }) +} diff --git a/src/domains/memory/repository.ts b/src/domains/memory/repository.ts index 3d36662c..9b44f871 100644 --- a/src/domains/memory/repository.ts +++ b/src/domains/memory/repository.ts @@ -7,6 +7,7 @@ import type { CapturedObservation } from "./observation-types" import { toDiffOperation, type ResolvedMemoryOperation } from "./resolve-operations" import { buildMemoryItemTokens } from "./search-index" import type { + FluidMemoryDeactivationReason, FluidMemoryKind, FluidMemoryPayload, MemoryDiffOperation, @@ -66,6 +67,24 @@ type MemoryRepository = { readonly deleteExpiredConsumedObservationsEffect: ( olderThan: Date, ) => Effect.Effect + readonly listActiveItemsEffect: ( + workspaceId: string, + ) => Effect.Effect< + readonly Pick[], + never, + DbClient + > + /** + * Move a specific set of active items to `inactive` with reason + * `decayed` (the activation-decay job's candidates, already confirmed by + * the caller — this never decides which items on its own). Mirrors the + * distill `deprecate` write path: drops the item's token rows so it stops + * surfacing as a dedup candidate. + */ + readonly deactivateDecayedItemsEffect: ( + workspaceId: string, + itemIds: readonly string[], + ) => Effect.Effect } type RawRowsResult = readonly Row[] | { readonly rows: readonly Row[] } @@ -257,6 +276,66 @@ const deleteExpiredConsumedObservationsEffect: MemoryRepository["deleteExpiredCo return deleted.length }) +const listActiveItemsEffect: MemoryRepository["listActiveItemsEffect"] = ( + workspaceId, +) => + Effect.gen(function* () { + const db = yield* DbClient + return yield* Effect.promise(() => + db + .select({ + id: fluidMemoryItems.id, + kind: fluidMemoryItems.kind, + createdAt: fluidMemoryItems.createdAt, + }) + .from(fluidMemoryItems) + .where( + and( + eq(fluidMemoryItems.workspaceId, workspaceId), + eq(fluidMemoryItems.status, "active"), + ), + ), + ) + }) + +const deactivateDecayedItemsEffect: MemoryRepository["deactivateDecayedItemsEffect"] = + (workspaceId, itemIds) => + Effect.gen(function* () { + if (itemIds.length === 0) return 0 + + const db = yield* DbClient + return yield* Effect.promise(() => + db.transaction(async (tx) => { + const updated = await tx + .update(fluidMemoryItems) + .set({ + status: "inactive", + deactivationReason: + "decayed" satisfies FluidMemoryDeactivationReason, + updatedAt: sql`now()`, + }) + .where( + and( + eq(fluidMemoryItems.workspaceId, workspaceId), + eq(fluidMemoryItems.status, "active"), + inArray(fluidMemoryItems.id, [...itemIds]), + ), + ) + .returning({ id: fluidMemoryItems.id }) + + if (updated.length > 0) { + await tx.delete(fluidMemoryTokens).where( + inArray( + fluidMemoryTokens.itemId, + updated.map((item) => item.id), + ), + ) + } + return updated.length + }), + ) + }) + export const memoryRepository: MemoryRepository = { findDedupCandidatesEffect, insertObservationsEffect, @@ -264,6 +343,8 @@ export const memoryRepository: MemoryRepository = { listPendingObservationsEffect, applyDistillBatchEffect, deleteExpiredConsumedObservationsEffect, + listActiveItemsEffect, + deactivateDecayedItemsEffect, } async function writeOperations( @@ -353,7 +434,8 @@ async function writeOperations( const [updated] = await tx .update(fluidMemoryItems) .set({ - status: "deprecated", + status: "inactive", + deactivationReason: "contradicted" satisfies FluidMemoryDeactivationReason, updatedAt: sql`now()`, }) .where( diff --git a/src/domains/memory/resolve-operations.test.ts b/src/domains/memory/resolve-operations.test.ts index 403a79a8..03484b9e 100644 --- a/src/domains/memory/resolve-operations.test.ts +++ b/src/domains/memory/resolve-operations.test.ts @@ -20,7 +20,7 @@ const existingItems = [ }, }, { id: "item-2", kind: "stance", status: "active" }, - { id: "item-3", kind: "stance", status: "deprecated" }, + { id: "item-3", kind: "stance", status: "inactive" }, { id: "item-4", kind: "entity_of_interest", @@ -140,7 +140,7 @@ describe("resolveMemoryOperations", () => { expect(resolved[0]?.op).toBe("skip") }) - it("downgrades merge to skip when the target is already deprecated", () => { + it("downgrades merge to skip when the target is already inactive", () => { const resolved = resolveMemoryOperations({ operations: makeOperations({ stances: [ diff --git a/src/domains/memory/service.ts b/src/domains/memory/service.ts index 41b8b5af..f711f98b 100644 --- a/src/domains/memory/service.ts +++ b/src/domains/memory/service.ts @@ -1,12 +1,14 @@ import "server-only" import { databaseRuntime } from "@/domains/workspace/database-runtime" +import { selectDecayCandidates, type DecayCandidate } from "./decay-candidates" import { memoryRepository, type ApplyDistillBatchInput, type InsertObservationsInput, } from "./repository" import type { FluidMemoryKind, MemoryDiffOperation } from "./types" +import { retrievalActivationService } from "@/domains/retrieval-activation/service" import type { FluidMemoryItem, FluidObservation, @@ -34,6 +36,20 @@ type MemoryService = { readonly deleteExpiredConsumedObservations: ( olderThan: Date, ) => Promise + /** + * Active items whose activation-decay score is below `scoreThreshold`. + * Read-only — does not change any item's status. The caller decides the + * threshold and what to do with the result (see + * `deactivateDecayedItems` to actually move candidates to `inactive`). + */ + readonly listDecayCandidates: ( + workspaceId: string, + options: { readonly now: Date; readonly scoreThreshold: number }, + ) => Promise + readonly deactivateDecayedItems: ( + workspaceId: string, + itemIds: readonly string[], + ) => Promise } const findDedupCandidates: MemoryService["findDedupCandidates"] = ( @@ -78,6 +94,46 @@ const deleteExpiredConsumedObservations: MemoryService["deleteExpiredConsumedObs memoryRepository.deleteExpiredConsumedObservationsEffect(olderThan), ) +const listDecayCandidates: MemoryService["listDecayCandidates"] = async ( + workspaceId, + options, +) => { + const items = await databaseRuntime.runPromise( + memoryRepository.listActiveItemsEffect(workspaceId), + ) + if (items.length === 0) return [] + + const activations = await retrievalActivationService.getActivations( + workspaceId, + "fluid_memory", + items.map((item) => item.id), + ) + const activationsById = new Map( + activations.map((activation) => [ + activation.unitRef, + { + activationCount: activation.activationCount, + lastActivatedAt: activation.lastActivatedAt, + }, + ]), + ) + + return selectDecayCandidates({ + items, + activationsById, + now: options.now, + scoreThreshold: options.scoreThreshold, + }) +} + +const deactivateDecayedItems: MemoryService["deactivateDecayedItems"] = ( + workspaceId, + itemIds, +) => + databaseRuntime.runPromise( + memoryRepository.deactivateDecayedItemsEffect(workspaceId, itemIds), + ) + export const memoryService: MemoryService = { findDedupCandidates, insertObservations, @@ -85,4 +141,6 @@ export const memoryService: MemoryService = { listPendingObservations, applyDistillBatch, deleteExpiredConsumedObservations, + listDecayCandidates, + deactivateDecayedItems, } diff --git a/src/domains/memory/types.ts b/src/domains/memory/types.ts index dac7602d..b9975cbd 100644 --- a/src/domains/memory/types.ts +++ b/src/domains/memory/types.ts @@ -91,6 +91,17 @@ export function parseFluidMemoryPayload( return result.success ? result.data : null } +/** + * Why an item left `active` for `inactive`. Orthogonal to `status`: `status` + * says whether the item is retrievable today, `deactivationReason` says + * which mechanism moved it out. + * - contradicted — distill decided a later turn reverses this item + * - decayed — the activation-decay job flagged it as unused past threshold + */ +export const fluidMemoryDeactivationReasons = ["contradicted", "decayed"] as const +export type FluidMemoryDeactivationReason = + (typeof fluidMemoryDeactivationReasons)[number] + /** One decided operation over the memory set; persisted into memory_diffs. */ export type MemoryDiffOperation = { readonly op: "create" | "skip" | "merge" | "deprecate" diff --git a/src/domains/retrieval-activation/decay-score.test.ts b/src/domains/retrieval-activation/decay-score.test.ts new file mode 100644 index 00000000..db239ebd --- /dev/null +++ b/src/domains/retrieval-activation/decay-score.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from "vitest" + +import { BASE_HALF_LIFE_DAYS, computeDecayScore } from "./decay-score" + +const NOW = new Date("2026-01-08T00:00:00Z") + +describe("computeDecayScore", () => { + it("scores a freshly anchored, never-activated unit as neutral 0.5", () => { + // freq = sigmoid(log1p(0)) = sigmoid(0) = 0.5; recency at age 0 = 1. + const score = computeDecayScore({ + activationCount: 0, + anchorAt: NOW, + now: NOW, + }) + expect(score).toBeCloseTo(0.5, 10) + }) + + it("halves the neutral score after one base half-life with no activations", () => { + const anchorAt = new Date( + NOW.getTime() - BASE_HALF_LIFE_DAYS * 24 * 60 * 60 * 1000, + ) + const score = computeDecayScore({ activationCount: 0, anchorAt, now: NOW }) + expect(score).toBeCloseTo(0.25, 10) + }) + + it("decays monotonically with age for a fixed activation count", () => { + const dayAgo = new Date(NOW.getTime() - 24 * 60 * 60 * 1000) + const weekAgo = new Date(NOW.getTime() - 7 * 24 * 60 * 60 * 1000) + + const scoreAtDay = computeDecayScore({ + activationCount: 2, + anchorAt: dayAgo, + now: NOW, + }) + const scoreAtWeek = computeDecayScore({ + activationCount: 2, + anchorAt: weekAgo, + now: NOW, + }) + + expect(scoreAtDay).toBeGreaterThan(scoreAtWeek) + }) + + it("scores a higher activation count above a lower one at the same age", () => { + const weekAgo = new Date(NOW.getTime() - 7 * 24 * 60 * 60 * 1000) + + const lowCount = computeDecayScore({ + activationCount: 1, + anchorAt: weekAgo, + now: NOW, + }) + const highCount = computeDecayScore({ + activationCount: 10, + anchorAt: weekAgo, + now: NOW, + }) + + expect(highCount).toBeGreaterThan(lowCount) + }) + + it("resets toward the frequency ceiling immediately after a fresh activation", () => { + // A unit with a long activation history but an activation just now + // should score close to its frequency ceiling, not its pre-reset decay. + const score = computeDecayScore({ + activationCount: 5, + anchorAt: NOW, + now: NOW, + }) + const frequencyCeiling = 1 / (1 + Math.exp(-Math.log1p(5))) + expect(score).toBeCloseTo(frequencyCeiling, 10) + }) + + it("stays within (0, 1) across a range of counts and ages", () => { + const activationCounts = [0, 1, 3, 10, 50] + const ageDaysList = [0, 1, 7, 30, 365] + + for (const activationCount of activationCounts) { + for (const ageDays of ageDaysList) { + const anchorAt = new Date( + NOW.getTime() - ageDays * 24 * 60 * 60 * 1000, + ) + const score = computeDecayScore({ activationCount, anchorAt, now: NOW }) + expect(score).toBeGreaterThan(0) + expect(score).toBeLessThan(1) + } + } + }) + + it("clamps negative age (anchor in the future) to zero elapsed time", () => { + const future = new Date(NOW.getTime() + 24 * 60 * 60 * 1000) + const score = computeDecayScore({ + activationCount: 0, + anchorAt: future, + now: NOW, + }) + expect(score).toBeCloseTo(0.5, 10) + }) +}) diff --git a/src/domains/retrieval-activation/decay-score.ts b/src/domains/retrieval-activation/decay-score.ts new file mode 100644 index 00000000..0e2ca1ba --- /dev/null +++ b/src/domains/retrieval-activation/decay-score.ts @@ -0,0 +1,66 @@ +/** + * Time-decay importance score for a retrievable unit (fluid memory item or + * crystal chunk). Pure function, computed at read time — never persisted — + * matching OpenViking's approach of not storing a score that would need + * migration whenever the formula changes. + * + * Structure copied from OpenViking's `hotness_score` + * (`sigmoid(log1p(activationCount)) × exp(-ln2/halfLife × ageDays)`), see + * `.repos/OpenViking/openviking/retrieve/memory_lifecycle.py`. + * + * `BASE_HALF_LIFE_DAYS` is deliberately 2x OpenViking's own default (7 + * days) — a longer grace period before an unused unit's importance + * meaningfully drops, confirmed against simulated day-counts (see the + * `记忆衰减聚类收尾方案` plan for the numbers this was checked against). + * + * One deviation from OpenViking: the recency half-life grows with + * `activationCount` instead of staying fixed, borrowing MemoryBank's + * (arXiv:2305.10250) intuition that repeated recall makes a memory more + * resistant to forgetting (there, strength `S` is incremented by 1 on every + * recall and used directly as the decay time constant). Here: + * + * effectiveHalfLifeDays = BASE_HALF_LIFE_DAYS * (1 + activationCount) + * + * This does not double-count activationCount with the frequency term: the + * frequency term sets the score's baseline ceiling for a given activation + * count, while the half-life growth slows how fast that ceiling erodes as + * time passes without a new activation. + * + * "Reset then decay" (an activation makes the unit feel fresh again, then + * importance decays again from there) is achieved by the caller advancing + * `anchorAt` to the activation time on every write — this function only + * computes the curve from whatever anchor it is given. + */ + +const MS_PER_DAY = 24 * 60 * 60 * 1000 + +/** 2x OpenViking's DEFAULT_HALF_LIFE_DAYS (7) — see rationale above. */ +export const BASE_HALF_LIFE_DAYS = 14 + +export type DecayScoreInput = { + /** Total times this unit has been cited into an answer. */ + readonly activationCount: number + /** Last activation time, or the unit's creation time if never activated. */ + readonly anchorAt: Date + readonly now: Date +} + +/** Always in (0, 1). */ +export function computeDecayScore(input: DecayScoreInput): number { + const activationCount = Math.max(input.activationCount, 0) + const ageDays = Math.max( + (input.now.getTime() - input.anchorAt.getTime()) / MS_PER_DAY, + 0, + ) + + const frequency = sigmoid(Math.log1p(activationCount)) + const effectiveHalfLifeDays = BASE_HALF_LIFE_DAYS * (1 + activationCount) + const decayRate = Math.LN2 / effectiveHalfLifeDays + const recency = Math.exp(-decayRate * ageDays) + + return frequency * recency +} + +function sigmoid(x: number): number { + return 1 / (1 + Math.exp(-x)) +} diff --git a/src/domains/retrieval-activation/repository.test.ts b/src/domains/retrieval-activation/repository.test.ts new file mode 100644 index 00000000..267c81e0 --- /dev/null +++ b/src/domains/retrieval-activation/repository.test.ts @@ -0,0 +1,138 @@ +import { afterEach, describe, expect, it, vi } from "vitest" +import { Effect, Layer } from "effect" + +import { retrievalActivationRepository } from "./repository" +import type { Db } from "@/infrastructure/db" + +type InsertValues = { + readonly workspaceId: string + readonly unitType: string + readonly unitRef: string + readonly activationCount: number +} + +afterEach(() => { + vi.restoreAllMocks() +}) + +async function runWithMockDb(insertValues: InsertValues[]) { + const insertBuilder = { + values: vi.fn((values: InsertValues[]) => { + insertValues.push(...values) + return insertBuilder + }), + onConflictDoUpdate: vi.fn(() => insertBuilder), + returning: vi.fn(async () => + insertValues.map((_, index) => ({ id: `activation_${index}` })), + ), + } + const dbMock = { insert: vi.fn(() => insertBuilder) } + const { DbClient } = await vi.importActual( + "@/infrastructure/db", + ) + const dbLayer = Layer.succeed(DbClient, dbMock as unknown as Db) + return { dbLayer, insertBuilder, dbMock } +} + +describe("retrievalActivationRepository.recordActivationsEffect", () => { + it("does nothing and never touches the db for an empty batch", async () => { + const insertValues: InsertValues[] = [] + const { dbLayer, dbMock } = await runWithMockDb(insertValues) + + const written = await Effect.runPromise( + retrievalActivationRepository + .recordActivationsEffect([]) + .pipe(Effect.provide(dbLayer)), + ) + + expect(written).toBe(0) + expect(dbMock.insert).not.toHaveBeenCalled() + }) + + it("collapses duplicate unit refs into a single upsert row", async () => { + const insertValues: InsertValues[] = [] + const { dbLayer, insertBuilder } = await runWithMockDb(insertValues) + + const written = await Effect.runPromise( + retrievalActivationRepository + .recordActivationsEffect([ + { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "doc:chunk_1" }, + { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "doc:chunk_1" }, + { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "doc:chunk_2" }, + ]) + .pipe(Effect.provide(dbLayer)), + ) + + expect(written).toBe(2) + expect(insertBuilder.values).toHaveBeenCalledOnce() + expect(insertValues).toHaveLength(2) + expect(insertValues.map((row) => row.unitRef).sort()).toEqual([ + "doc:chunk_1", + "doc:chunk_2", + ]) + }) + + it("keeps the same unit ref distinct across different unit types and workspaces", async () => { + const insertValues: InsertValues[] = [] + const { dbLayer } = await runWithMockDb(insertValues) + + await Effect.runPromise( + retrievalActivationRepository + .recordActivationsEffect([ + { workspaceId: "ws_1", unitType: "fluid_memory", unitRef: "item_1" }, + { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "item_1" }, + { workspaceId: "ws_2", unitType: "fluid_memory", unitRef: "item_1" }, + ]) + .pipe(Effect.provide(dbLayer)), + ) + + expect(insertValues).toHaveLength(3) + }) +}) + +describe("retrievalActivationRepository.getActivationsEffect", () => { + it("returns [] without querying the db for an empty unit ref list", async () => { + const selectBuilder = { from: vi.fn(), where: vi.fn() } + const dbMock = { select: vi.fn(() => selectBuilder) } + const { DbClient } = await vi.importActual( + "@/infrastructure/db", + ) + const dbLayer = Layer.succeed(DbClient, dbMock as unknown as Db) + + const result = await Effect.runPromise( + retrievalActivationRepository + .getActivationsEffect("ws_1", "fluid_memory", []) + .pipe(Effect.provide(dbLayer)), + ) + + expect(result).toEqual([]) + expect(dbMock.select).not.toHaveBeenCalled() + }) + + it("returns matching activation rows", async () => { + const rows = [ + { + unitRef: "item_1", + activationCount: 3, + lastActivatedAt: new Date("2026-01-01T00:00:00Z"), + }, + ] + const selectBuilder = { + from: vi.fn(() => selectBuilder), + where: vi.fn(async () => rows), + } + const dbMock = { select: vi.fn(() => selectBuilder) } + const { DbClient } = await vi.importActual( + "@/infrastructure/db", + ) + const dbLayer = Layer.succeed(DbClient, dbMock as unknown as Db) + + const result = await Effect.runPromise( + retrievalActivationRepository + .getActivationsEffect("ws_1", "fluid_memory", ["item_1", "item_2"]) + .pipe(Effect.provide(dbLayer)), + ) + + expect(result).toEqual(rows) + }) +}) diff --git a/src/domains/retrieval-activation/repository.ts b/src/domains/retrieval-activation/repository.ts new file mode 100644 index 00000000..53732acb --- /dev/null +++ b/src/domains/retrieval-activation/repository.ts @@ -0,0 +1,119 @@ +import "server-only" + +import { and, eq, inArray, sql } from "drizzle-orm" +import { Effect } from "effect" + +import type { RetrievalUnitType } from "./types" +import { DbClient } from "@/infrastructure/db" +import { retrievalActivations } from "@/infrastructure/db/schema" + +export type RecordActivationInput = { + readonly workspaceId: string + readonly unitType: RetrievalUnitType + readonly unitRef: string +} + +export type ActivationStats = { + readonly unitRef: string + readonly activationCount: number + readonly lastActivatedAt: Date | null +} + +type RetrievalActivationRepository = { + /** + * Upsert one activation (+1, lastActivatedAt = now) per distinct unit. + * Duplicate (workspaceId, unitType, unitRef) entries within `inputs` are + * collapsed to a single +1 — Postgres rejects a multi-row upsert that + * would touch the same conflict target twice in one statement, and + * "cited twice in one answer" should still only count as one activation + * event for this turn. + */ + readonly recordActivationsEffect: ( + inputs: readonly RecordActivationInput[], + ) => Effect.Effect + /** + * Existing ledger rows for a set of unit refs of one type. Units with no + * row (never activated) are simply absent from the result — the caller + * treats that as activationCount 0. + */ + readonly getActivationsEffect: ( + workspaceId: string, + unitType: RetrievalUnitType, + unitRefs: readonly string[], + ) => Effect.Effect +} + +const recordActivationsEffect: RetrievalActivationRepository["recordActivationsEffect"] = + (inputs) => + Effect.gen(function* () { + const deduped = dedupeInputs(inputs) + if (deduped.length === 0) return 0 + + const db = yield* DbClient + const written = yield* Effect.promise(() => + db + .insert(retrievalActivations) + .values( + deduped.map((input) => ({ + workspaceId: input.workspaceId, + unitType: input.unitType, + unitRef: input.unitRef, + activationCount: 1, + lastActivatedAt: sql`now()`, + })), + ) + .onConflictDoUpdate({ + target: [ + retrievalActivations.workspaceId, + retrievalActivations.unitType, + retrievalActivations.unitRef, + ], + set: { + activationCount: sql`${retrievalActivations.activationCount} + 1`, + lastActivatedAt: sql`now()`, + }, + }) + .returning({ id: retrievalActivations.id }), + ) + return written.length + }) + +const getActivationsEffect: RetrievalActivationRepository["getActivationsEffect"] = + (workspaceId, unitType, unitRefs) => + Effect.gen(function* () { + if (unitRefs.length === 0) return [] + + const db = yield* DbClient + const rows = yield* Effect.promise(() => + db + .select({ + unitRef: retrievalActivations.unitRef, + activationCount: retrievalActivations.activationCount, + lastActivatedAt: retrievalActivations.lastActivatedAt, + }) + .from(retrievalActivations) + .where( + and( + eq(retrievalActivations.workspaceId, workspaceId), + eq(retrievalActivations.unitType, unitType), + inArray(retrievalActivations.unitRef, [...unitRefs]), + ), + ), + ) + return rows + }) + +export const retrievalActivationRepository: RetrievalActivationRepository = { + recordActivationsEffect, + getActivationsEffect, +} + +function dedupeInputs( + inputs: readonly RecordActivationInput[], +): RecordActivationInput[] { + const byKey = new Map() + for (const input of inputs) { + byKey.set(`${input.workspaceId}\u0000${input.unitType}\u0000${input.unitRef}`, input) + } + return [...byKey.values()] +} diff --git a/src/domains/retrieval-activation/service.ts b/src/domains/retrieval-activation/service.ts new file mode 100644 index 00000000..990a6eba --- /dev/null +++ b/src/domains/retrieval-activation/service.ts @@ -0,0 +1,45 @@ +import "server-only" + +import { + retrievalActivationRepository, + type ActivationStats, + type RecordActivationInput, +} from "./repository" +import type { RetrievalUnitType } from "./types" +import { databaseRuntime } from "@/domains/workspace/database-runtime" + +type RetrievalActivationService = { + readonly recordActivations: ( + inputs: readonly RecordActivationInput[], + ) => Promise + readonly getActivations: ( + workspaceId: string, + unitType: RetrievalUnitType, + unitRefs: readonly string[], + ) => Promise +} + +const recordActivations: RetrievalActivationService["recordActivations"] = ( + inputs, +) => + databaseRuntime.runPromise( + retrievalActivationRepository.recordActivationsEffect(inputs), + ) + +const getActivations: RetrievalActivationService["getActivations"] = ( + workspaceId, + unitType, + unitRefs, +) => + databaseRuntime.runPromise( + retrievalActivationRepository.getActivationsEffect( + workspaceId, + unitType, + unitRefs, + ), + ) + +export const retrievalActivationService: RetrievalActivationService = { + recordActivations, + getActivations, +} diff --git a/src/domains/retrieval-activation/types.ts b/src/domains/retrieval-activation/types.ts new file mode 100644 index 00000000..29e18db1 --- /dev/null +++ b/src/domains/retrieval-activation/types.ts @@ -0,0 +1,17 @@ +/** + * A "unit" is anything retrieval can surface and an answer can actually + * cite. Today there are two kinds: + * - fluid_memory — a `fluid_memory_items` row, keyed by its id + * - crystal_chunk — a Knowhere chunk, which has no local row; keyed by + * `${documentId}:${chunkId}` (see `toChunkUnitRef`) + */ +export const retrievalUnitTypes = ["fluid_memory", "crystal_chunk"] as const +export type RetrievalUnitType = (typeof retrievalUnitTypes)[number] + +/** Composite key for a crystal_chunk unit ref. */ +export function toChunkUnitRef(input: { + readonly documentId: string + readonly chunkId: string +}): string { + return `${input.documentId}:${input.chunkId}` +} diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index cb767a02..8963f95d 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -358,9 +358,16 @@ export type NewChatMessage = typeof chatMessages.$inferInsert; * one line for pre-filter/dedup context, L1 = short paragraph for later * cognition injection). L2 is the payload itself. * - * Lifecycle: rows start `active`; user revisions deprecate rather than + * Lifecycle: rows start `active`; user revisions deactivate rather than * delete (conservative merge policy), with `version` bumped on merge. * + * `status` is `active` | `inactive`. `inactive` is not itself a + * disambiguated state — `deactivation_reason` records why an item left + * `active` (e.g. `contradicted`, when distill decides a new turn reverses + * this item). This keeps the decay/lifecycle axis (`status`) separate from + * the reason axis, so an activation-decay job can later flip items to + * `inactive` with a different reason without inventing a new status value. + * * `source_message_id` points at the assistant message of the turn the * insight was extracted from; it is set-null on message deletion because * the insight outlives any single turn. @@ -382,6 +389,7 @@ export const fluidMemoryItems = pgTable( ), confidence: doublePrecision("confidence").notNull(), status: text("status").notNull(), + deactivationReason: text("deactivation_reason"), version: integer("version").notNull().default(1), createdAt: timestamp("created_at", { withTimezone: true }) .notNull() @@ -391,7 +399,7 @@ export const fluidMemoryItems = pgTable( .defaultNow(), }, (t) => [ - // Workspace lifecycle scans (active vs deprecated). + // Workspace lifecycle scans (active vs inactive). index("fluid_memory_items_workspace_status_idx").on( t.workspaceId, t.status, @@ -411,8 +419,8 @@ export type NewFluidMemoryItem = typeof fluidMemoryItems.$inferInsert; * * Invariant: token rows exist iff the owning item is `active`. Writers keep * this in sync — create inserts rows, merge replaces them, deprecate deletes - * them — so lookups scan tokens alone (no status join) and never surface a - * deprecated item. + * them — so lookups scan tokens alone (no status join) and never surface an + * inactive item. * * Tokenization mirrors Knowhere map-nav: single CJK characters plus * `[a-z0-9_]+` runs. Scoring is idf-weighted token overlap computed in SQL, @@ -526,3 +534,47 @@ export const fluidObservations = pgTable( export type FluidObservation = typeof fluidObservations.$inferSelect; export type NewFluidObservation = typeof fluidObservations.$inferInsert; + +/** + * Unified activation ledger for retrievable units, driving time-decay + * importance (see src/domains/retrieval-activation/decay-score.ts). + * + * A "unit" is anything that can be surfaced by retrieval and actually cited + * into an answer: today `fluid_memory` (a `fluid_memory_items` row, keyed by + * its id) and `crystal_chunk` (a Knowhere chunk, which has no local row — + * keyed by `${documentId}:${chunkId}`, composed at write time). + * + * Only "really used in an answer" writes here (a citation), not "entered + * the candidate pool" — this avoids overcounting recall as usage. + * + * For `crystal_chunk`, `created_at` is this row's first-write time (the + * first time Notebook observed this chunk being cited), not the chunk's + * true ingestion time in Knowhere — that timestamp is not available to + * Notebook. This is a known, deliberate approximation. + */ +export const retrievalActivations = pgTable( + "retrieval_activations", + { + id: uuid("id").primaryKey().defaultRandom(), + workspaceId: uuid("workspace_id") + .notNull() + .references(() => workspaces.id, { onDelete: "cascade" }), + unitType: text("unit_type").notNull(), + unitRef: text("unit_ref").notNull(), + activationCount: integer("activation_count").notNull().default(0), + lastActivatedAt: timestamp("last_activated_at", { withTimezone: true }), + createdAt: timestamp("created_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (t) => [ + uniqueIndex("retrieval_activations_unit_idx").on( + t.workspaceId, + t.unitType, + t.unitRef, + ), + ], +); + +export type RetrievalActivation = typeof retrievalActivations.$inferSelect; +export type NewRetrievalActivation = typeof retrievalActivations.$inferInsert; From 2d46bc07112d0678d69e0dcd0682401329ecc29e Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Thu, 10 Sep 2026 16:00:33 +0800 Subject: [PATCH 08/11] feat: update environment configuration and integrate new SDK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Updated `.env.local.example` to reflect the new Knowhere API base URL and added Memento service configuration. - Added `@modelcontextprotocol/sdk` dependency to `package.json` and updated `pnpm-lock.yaml` accordingly. - Removed deprecated constants and scripts related to the 观心 case, streamlining the codebase. - Refactored agent harness to utilize the new memory capture functionality, enhancing memory management during chat interactions. - Updated tests to reflect changes in memory handling and ensure proper integration with the new SDK. This update improves the overall functionality and maintainability of the memory and chat systems. --- .env.local.example | 21 +- drizzle/0017_overjoyed_shooting_star.sql | 5 + drizzle/meta/0017_snapshot.json | 932 ++++++++++++++++++ drizzle/meta/_journal.json | 7 + next.config.ts | 1 + package.json | 1 + pnpm-lock.yaml | 460 +++++---- scripts/guanxin-case/constants.ts | 8 - scripts/guanxin-case/ensure-workspace.mts | 21 - scripts/guanxin-case/sample-pilot.py | 91 -- src/agent-harness/runtime.test.ts | 26 +- src/agent-harness/runtime.ts | 27 +- src/app/api/memory/distill/route.ts | 27 - src/app/api/memory/extract/route.ts | 27 - src/domains/chat/commit-turn.test.ts | 25 +- src/domains/chat/commit-turn.ts | 68 +- src/domains/chat/memory-tools.test.ts | 104 -- src/domains/chat/memory-tools.ts | 66 -- src/domains/chat/prompt.ts | 4 +- src/domains/chat/route-service.test.ts | 17 +- src/domains/chat/service.test.ts | 32 +- src/domains/chat/service.ts | 2 +- src/domains/memory/decay-candidates.test.ts | 69 -- src/domains/memory/decay-candidates.ts | 64 -- src/domains/memory/distill-config.ts | 19 - src/domains/memory/distill-model.ts | 83 -- src/domains/memory/distill-prompts.test.ts | 110 --- src/domains/memory/distill-prompts.ts | 311 ------ src/domains/memory/distill-trigger.test.ts | 84 -- src/domains/memory/distill-trigger.ts | 77 -- src/domains/memory/distill-types.test.ts | 209 ---- src/domains/memory/distill-types.ts | 190 ---- src/domains/memory/distill-workflow.test.ts | 48 - src/domains/memory/distill-workflow.ts | 231 ----- src/domains/memory/extract-trigger.ts | 43 - src/domains/memory/extract-workflow.test.ts | 40 - src/domains/memory/extract-workflow.ts | 128 --- src/domains/memory/extraction-model.ts | 50 - src/domains/memory/observation-types.test.ts | 78 -- src/domains/memory/observation-types.ts | 50 - src/domains/memory/prompts.test.ts | 54 - src/domains/memory/prompts.ts | 102 -- src/domains/memory/repository.ts | 500 ---------- src/domains/memory/resolve-operations.test.ts | 378 ------- src/domains/memory/resolve-operations.ts | 314 ------ src/domains/memory/search-index.test.ts | 94 -- src/domains/memory/search-index.ts | 84 -- src/domains/memory/service.ts | 146 --- src/domains/memory/types.ts | 112 --- .../retrieval-activation/decay-score.test.ts | 98 -- .../retrieval-activation/decay-score.ts | 66 -- .../retrieval-activation/repository.test.ts | 138 --- .../retrieval-activation/repository.ts | 119 --- src/domains/retrieval-activation/service.ts | 45 - src/domains/retrieval-activation/types.ts | 17 - src/infrastructure/db/schema.ts | 240 ----- src/integrations/memento/client.ts | 75 ++ src/integrations/memento/config.ts | 18 + src/integrations/memento/memory-tools.ts | 105 ++ 59 files changed, 1499 insertions(+), 5062 deletions(-) create mode 100644 drizzle/0017_overjoyed_shooting_star.sql create mode 100644 drizzle/meta/0017_snapshot.json delete mode 100644 scripts/guanxin-case/constants.ts delete mode 100644 scripts/guanxin-case/ensure-workspace.mts delete mode 100644 scripts/guanxin-case/sample-pilot.py delete mode 100644 src/app/api/memory/distill/route.ts delete mode 100644 src/app/api/memory/extract/route.ts delete mode 100644 src/domains/chat/memory-tools.test.ts delete mode 100644 src/domains/chat/memory-tools.ts delete mode 100644 src/domains/memory/decay-candidates.test.ts delete mode 100644 src/domains/memory/decay-candidates.ts delete mode 100644 src/domains/memory/distill-config.ts delete mode 100644 src/domains/memory/distill-model.ts delete mode 100644 src/domains/memory/distill-prompts.test.ts delete mode 100644 src/domains/memory/distill-prompts.ts delete mode 100644 src/domains/memory/distill-trigger.test.ts delete mode 100644 src/domains/memory/distill-trigger.ts delete mode 100644 src/domains/memory/distill-types.test.ts delete mode 100644 src/domains/memory/distill-types.ts delete mode 100644 src/domains/memory/distill-workflow.test.ts delete mode 100644 src/domains/memory/distill-workflow.ts delete mode 100644 src/domains/memory/extract-trigger.ts delete mode 100644 src/domains/memory/extract-workflow.test.ts delete mode 100644 src/domains/memory/extract-workflow.ts delete mode 100644 src/domains/memory/extraction-model.ts delete mode 100644 src/domains/memory/observation-types.test.ts delete mode 100644 src/domains/memory/observation-types.ts delete mode 100644 src/domains/memory/prompts.test.ts delete mode 100644 src/domains/memory/prompts.ts delete mode 100644 src/domains/memory/repository.ts delete mode 100644 src/domains/memory/resolve-operations.test.ts delete mode 100644 src/domains/memory/resolve-operations.ts delete mode 100644 src/domains/memory/search-index.test.ts delete mode 100644 src/domains/memory/search-index.ts delete mode 100644 src/domains/memory/service.ts delete mode 100644 src/domains/memory/types.ts delete mode 100644 src/domains/retrieval-activation/decay-score.test.ts delete mode 100644 src/domains/retrieval-activation/decay-score.ts delete mode 100644 src/domains/retrieval-activation/repository.test.ts delete mode 100644 src/domains/retrieval-activation/repository.ts delete mode 100644 src/domains/retrieval-activation/service.ts delete mode 100644 src/domains/retrieval-activation/types.ts create mode 100644 src/integrations/memento/client.ts create mode 100644 src/integrations/memento/config.ts create mode 100644 src/integrations/memento/memory-tools.ts diff --git a/.env.local.example b/.env.local.example index 13078722..097692fe 100644 --- a/.env.local.example +++ b/.env.local.example @@ -1,11 +1,17 @@ -# Optional API base URL. Defaults to production when unset. -# Use staging when validating staging keys. -# KNOWHERE_BASE_URL=https://api-staging.knowhereto.ai +# Knowhere API origin. Unset also means production. +# Local recommendation: production. Staging is only for staging keys. +KNOWHERE_BASE_URL=https://api.knowhereto.ai -# Optional development override. When set, Notebook skips Dashboard session +# Local development override. When set, Notebook skips Dashboard session # auth and Dashboard-issued JWT creation, then calls Knowhere directly with # this key. Leave unset for production and Dashboard-authenticated staging. -# KNOWHERE_API_KEY=sk_your_development_key_here +# KNOWHERE_API_KEY= + +# --- Local demo case only (观心). Not Notebook core. --- +# Put the real key in gitignored `.env.local`, copied from +# `观心2.0-RAG-v1.1-demo/_retrieval/.env`. +# Do not copy that file's KNOWHERE_BASE_URL — it is a retrieval endpoint +# path, not the SDK origin above. # --- Chat provider (server-side only) --- # Vercel AI Gateway key; AI SDK picks it up automatically @@ -61,3 +67,8 @@ DATABASE_URL=postgres://user:password@host/db # pg — postgres-js (use for local dev against a plain Postgres, # or for AWS Aurora Postgres if/when we migrate off Neon) DATABASE_DRIVER=pg + +# --- Memento (fluid memory + retrieval activation) --- +# Standalone REST + MCP service. Notebook no longer owns these tables. +MEMENTO_BASE_URL=http://localhost:8787 +MEMENTO_SERVICE_KEY= diff --git a/drizzle/0017_overjoyed_shooting_star.sql b/drizzle/0017_overjoyed_shooting_star.sql new file mode 100644 index 00000000..6d97d3ac --- /dev/null +++ b/drizzle/0017_overjoyed_shooting_star.sql @@ -0,0 +1,5 @@ +DROP TABLE "fluid_memory_items" CASCADE;--> statement-breakpoint +DROP TABLE "fluid_memory_tokens" CASCADE;--> statement-breakpoint +DROP TABLE "fluid_observations" CASCADE;--> statement-breakpoint +DROP TABLE "memory_diffs" CASCADE;--> statement-breakpoint +DROP TABLE "retrieval_activations" CASCADE; \ No newline at end of file diff --git a/drizzle/meta/0017_snapshot.json b/drizzle/meta/0017_snapshot.json new file mode 100644 index 00000000..52d88234 --- /dev/null +++ b/drizzle/meta/0017_snapshot.json @@ -0,0 +1,932 @@ +{ + "id": "d4de5ce8-e16b-4dc2-a9ad-2b76bbca8150", + "prevId": "55d73dc7-9826-4cb5-9bb6-3506b8fa338a", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parsed_document_sync_leases": { + "name": "parsed_document_sync_leases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_token": { + "name": "lease_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_reason": { + "name": "release_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parsed_document_sync_leases_token_idx": { + "name": "parsed_document_sync_leases_token_idx", + "columns": [ + { + "expression": "lease_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_active_idx": { + "name": "parsed_document_sync_leases_active_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_workspace_active_idx": { + "name": "parsed_document_sync_leases_workspace_active_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_document_active_idx": { + "name": "parsed_document_sync_leases_document_active_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parsed_document_sync_leases_workspace_id_workspaces_id_fk": { + "name": "parsed_document_sync_leases_workspace_id_workspaces_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parsed_document_sync_leases_source_id_sources_id_fk": { + "name": "parsed_document_sync_leases_source_id_sources_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index a8f7671e..4ee28af6 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -120,6 +120,13 @@ "when": 1788952172906, "tag": "0016_wealthy_matthew_murdock", "breakpoints": true + }, + { + "idx": 17, + "version": "7", + "when": 1789017481605, + "tag": "0017_overjoyed_shooting_star", + "breakpoints": true } ] } \ No newline at end of file diff --git a/next.config.ts b/next.config.ts index b73f5201..cd1fac0a 100644 --- a/next.config.ts +++ b/next.config.ts @@ -10,6 +10,7 @@ const nextConfig: NextConfig = { "@ontos-ai/knowhere-sdk", "@napi-rs/canvas", "piscina", + "@modelcontextprotocol/sdk", ], allowedDevOrigins: [ "127.0.0.1", diff --git a/package.json b/package.json index f3778893..4ab25322 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "@ai-sdk/react": "^3.0.177", "@antv/chart-visualization-skills": "0.1.3", "@effect/platform": "^0.96.1", + "@modelcontextprotocol/sdk": "^1.30.0", "@napi-rs/canvas": "^1.0.2", "@neondatabase/serverless": "^1.1.0", "@ontos-ai/knowhere-sdk": "^2.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8847510a..d4db84c2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: '@effect/platform': specifier: ^0.96.1 version: 0.96.1(effect@3.21.2) + '@modelcontextprotocol/sdk': + specifier: ^1.30.0 + version: 1.30.0(supports-color@7.2.0)(zod@4.4.3) '@napi-rs/canvas': specifier: ^1.0.2 version: 1.0.2 @@ -25,7 +28,7 @@ importers: version: 1.1.0 '@ontos-ai/knowhere-sdk': specifier: ^2.2.0 - version: 2.2.0 + version: 2.2.0(debug@4.4.3(supports-color@7.2.0))(supports-color@7.2.0) '@radix-ui/react-alert-dialog': specifier: ^1.1.15 version: 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -97,7 +100,7 @@ importers: version: 1.12.0 next: specifier: 16.2.4 - version: 16.2.4(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + version: 16.2.4(@babel/core@7.29.0(supports-color@7.2.0))(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) next-themes: specifier: ^0.4.6 version: 0.4.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -121,13 +124,13 @@ importers: version: 19.2.4(react@19.2.4) react-markdown: specifier: ^10.1.0 - version: 10.1.0(@types/react@19.2.14)(react@19.2.4) + version: 10.1.0(@types/react@19.2.14)(react@19.2.4)(supports-color@7.2.0) react-pdf: specifier: ^10.4.1 version: 10.4.1(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) remark-gfm: specifier: ^4.0.1 - version: 4.0.1 + version: 4.0.1(supports-color@7.2.0) server-only: specifier: ^0.0.1 version: 0.0.1 @@ -182,16 +185,16 @@ importers: version: 0.31.10 eslint: specifier: ^9 - version: 9.39.4(jiti@2.7.0) + version: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) eslint-config-next: specifier: 16.2.4 - version: 16.2.4(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) + version: 16.2.4(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) jsdom: specifier: ^29.1.1 version: 29.1.1(@noble/hashes@1.8.0) shadcn: specifier: ^4.7.0 - version: 4.7.0(@types/node@20.19.39)(typescript@6.0.3) + version: 4.7.0(@types/node@20.19.39)(supports-color@7.2.0)(typescript@6.0.3) tailwindcss: specifier: ^4 version: 4.2.4 @@ -455,11 +458,11 @@ packages: '@esbuild-kit/core-utils@3.3.2': resolution: {integrity: sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild-kit/esm-loader@2.6.5': resolution: {integrity: sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild/aix-ppc64@0.25.12': resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} @@ -1197,8 +1200,8 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} - '@modelcontextprotocol/sdk@1.29.0': - resolution: {integrity: sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==} + '@modelcontextprotocol/sdk@1.30.0': + resolution: {integrity: sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==} engines: {node: '>=18'} peerDependencies: '@cfworker/json-schema': ^4.1.1 @@ -2594,6 +2597,7 @@ packages: '@xmldom/xmldom@0.8.13': resolution: {integrity: sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==} engines: {node: '>=10.0.0'} + deprecated: this version has critical issues, please update to the latest version accepts@2.0.0: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} @@ -2948,6 +2952,7 @@ packages: crypto-js@4.2.0: resolution: {integrity: sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==} + deprecated: Active development of CryptoJS has been discontinued. This library is no longer maintained. css-tree@3.2.1: resolution: {integrity: sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==} @@ -3394,6 +3399,7 @@ packages: eslint@9.39.4: resolution: {integrity: sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true peerDependencies: jiti: '*' @@ -5733,20 +5739,20 @@ snapshots: '@babel/compat-data@7.29.3': {} - '@babel/core@7.29.0': + '@babel/core@7.29.0(supports-color@7.2.0)': dependencies: '@babel/code-frame': 7.29.0 '@babel/generator': 7.29.1 '@babel/helper-compilation-targets': 7.28.6 - '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0) + '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helpers': 7.29.2 '@babel/parser': 7.29.3 '@babel/template': 7.28.6 - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) '@babel/types': 7.29.0 '@jridgewell/remapping': 2.3.5 convert-source-map: 2.0.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -5773,41 +5779,41 @@ snapshots: lru-cache: 5.1.1 semver: 6.3.1 - '@babel/helper-create-class-features-plugin@7.29.3(@babel/core@7.29.0)': + '@babel/helper-create-class-features-plugin@7.29.3(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/helper-annotate-as-pure': 7.27.3 - '@babel/helper-member-expression-to-functions': 7.28.5 + '@babel/helper-member-expression-to-functions': 7.28.5(supports-color@7.2.0) '@babel/helper-optimise-call-expression': 7.27.1 - '@babel/helper-replace-supers': 7.28.6(@babel/core@7.29.0) - '@babel/helper-skip-transparent-expression-wrappers': 7.27.1 - '@babel/traverse': 7.29.0 + '@babel/helper-replace-supers': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/helper-skip-transparent-expression-wrappers': 7.27.1(supports-color@7.2.0) + '@babel/traverse': 7.29.0(supports-color@7.2.0) semver: 6.3.1 transitivePeerDependencies: - supports-color '@babel/helper-globals@7.28.0': {} - '@babel/helper-member-expression-to-functions@7.28.5': + '@babel/helper-member-expression-to-functions@7.28.5(supports-color@7.2.0)': dependencies: - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) '@babel/types': 7.29.0 transitivePeerDependencies: - supports-color - '@babel/helper-module-imports@7.28.6': + '@babel/helper-module-imports@7.28.6(supports-color@7.2.0)': dependencies: - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) '@babel/types': 7.29.0 transitivePeerDependencies: - supports-color - '@babel/helper-module-transforms@7.28.6(@babel/core@7.29.0)': + '@babel/helper-module-transforms@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 - '@babel/helper-module-imports': 7.28.6 + '@babel/core': 7.29.0(supports-color@7.2.0) + '@babel/helper-module-imports': 7.28.6(supports-color@7.2.0) '@babel/helper-validator-identifier': 7.28.5 - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -5817,18 +5823,18 @@ snapshots: '@babel/helper-plugin-utils@7.28.6': {} - '@babel/helper-replace-supers@7.28.6(@babel/core@7.29.0)': + '@babel/helper-replace-supers@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 - '@babel/helper-member-expression-to-functions': 7.28.5 + '@babel/core': 7.29.0(supports-color@7.2.0) + '@babel/helper-member-expression-to-functions': 7.28.5(supports-color@7.2.0) '@babel/helper-optimise-call-expression': 7.27.1 - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) transitivePeerDependencies: - supports-color - '@babel/helper-skip-transparent-expression-wrappers@7.27.1': + '@babel/helper-skip-transparent-expression-wrappers@7.27.1(supports-color@7.2.0)': dependencies: - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) '@babel/types': 7.29.0 transitivePeerDependencies: - supports-color @@ -5848,43 +5854,43 @@ snapshots: dependencies: '@babel/types': 7.29.0 - '@babel/plugin-syntax-jsx@7.28.6(@babel/core@7.29.0)': + '@babel/plugin-syntax-jsx@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))': dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.28.6 - '@babel/plugin-syntax-typescript@7.28.6(@babel/core@7.29.0)': + '@babel/plugin-syntax-typescript@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))': dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.28.6 - '@babel/plugin-transform-modules-commonjs@7.28.6(@babel/core@7.29.0)': + '@babel/plugin-transform-modules-commonjs@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 - '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0) + '@babel/core': 7.29.0(supports-color@7.2.0) + '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.28.6 transitivePeerDependencies: - supports-color - '@babel/plugin-transform-typescript@7.28.6(@babel/core@7.29.0)': + '@babel/plugin-transform-typescript@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/helper-annotate-as-pure': 7.27.3 - '@babel/helper-create-class-features-plugin': 7.29.3(@babel/core@7.29.0) + '@babel/helper-create-class-features-plugin': 7.29.3(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.28.6 - '@babel/helper-skip-transparent-expression-wrappers': 7.27.1 - '@babel/plugin-syntax-typescript': 7.28.6(@babel/core@7.29.0) + '@babel/helper-skip-transparent-expression-wrappers': 7.27.1(supports-color@7.2.0) + '@babel/plugin-syntax-typescript': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0)) transitivePeerDependencies: - supports-color - '@babel/preset-typescript@7.28.5(@babel/core@7.29.0)': + '@babel/preset-typescript@7.28.5(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.28.6 '@babel/helper-validator-option': 7.27.1 - '@babel/plugin-syntax-jsx': 7.28.6(@babel/core@7.29.0) - '@babel/plugin-transform-modules-commonjs': 7.28.6(@babel/core@7.29.0) - '@babel/plugin-transform-typescript': 7.28.6(@babel/core@7.29.0) + '@babel/plugin-syntax-jsx': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0)) + '@babel/plugin-transform-modules-commonjs': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/plugin-transform-typescript': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -5896,7 +5902,7 @@ snapshots: '@babel/parser': 7.29.3 '@babel/types': 7.29.0 - '@babel/traverse@7.29.0': + '@babel/traverse@7.29.0(supports-color@7.2.0)': dependencies: '@babel/code-frame': 7.29.0 '@babel/generator': 7.29.1 @@ -5904,7 +5910,7 @@ snapshots: '@babel/parser': 7.29.3 '@babel/template': 7.28.6 '@babel/types': 7.29.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -6217,17 +6223,17 @@ snapshots: '@esbuild/win32-x64@0.27.7': optional: true - '@eslint-community/eslint-utils@4.9.1(eslint@9.39.4(jiti@2.7.0))': + '@eslint-community/eslint-utils@4.9.1(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))': dependencies: - eslint: 9.39.4(jiti@2.7.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) eslint-visitor-keys: 3.4.3 '@eslint-community/regexpp@4.12.2': {} - '@eslint/config-array@0.21.2': + '@eslint/config-array@0.21.2(supports-color@7.2.0)': dependencies: '@eslint/object-schema': 2.1.7 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) minimatch: 3.1.5 transitivePeerDependencies: - supports-color @@ -6240,10 +6246,10 @@ snapshots: dependencies: '@types/json-schema': 7.0.15 - '@eslint/eslintrc@3.3.5': + '@eslint/eslintrc@3.3.5(supports-color@7.2.0)': dependencies: ajv: 6.15.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) espree: 10.4.0 globals: 14.0.0 ignore: 5.3.2 @@ -6447,7 +6453,7 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 - '@modelcontextprotocol/sdk@1.29.0(zod@3.25.76)': + '@modelcontextprotocol/sdk@1.30.0(supports-color@7.2.0)(zod@3.25.76)': dependencies: '@hono/node-server': 1.19.14(hono@4.12.17) ajv: 8.20.0 @@ -6457,8 +6463,8 @@ snapshots: cross-spawn: 7.0.6 eventsource: 3.0.7 eventsource-parser: 3.0.8 - express: 5.2.1 - express-rate-limit: 8.5.0(express@5.2.1) + express: 5.2.1(supports-color@7.2.0) + express-rate-limit: 8.5.0(express@5.2.1(supports-color@7.2.0)) hono: 4.12.17 jose: 6.2.3 json-schema-typed: 8.0.2 @@ -6469,6 +6475,28 @@ snapshots: transitivePeerDependencies: - supports-color + '@modelcontextprotocol/sdk@1.30.0(supports-color@7.2.0)(zod@4.4.3)': + dependencies: + '@hono/node-server': 1.19.14(hono@4.12.17) + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) + content-type: 1.0.5 + cors: 2.8.6 + cross-spawn: 7.0.6 + eventsource: 3.0.7 + eventsource-parser: 3.0.8 + express: 5.2.1(supports-color@7.2.0) + express-rate-limit: 8.5.0(express@5.2.1(supports-color@7.2.0)) + hono: 4.12.17 + jose: 6.2.3 + json-schema-typed: 8.0.2 + pkce-challenge: 5.0.1 + raw-body: 3.0.2 + zod: 4.4.3 + zod-to-json-schema: 3.25.2(zod@4.4.3) + transitivePeerDependencies: + - supports-color + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': optional: true @@ -6731,9 +6759,9 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} - '@ontos-ai/knowhere-sdk@2.2.0': + '@ontos-ai/knowhere-sdk@2.2.0(debug@4.4.3(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - axios: 1.18.1 + axios: 1.18.1(debug@4.4.3(supports-color@7.2.0))(supports-color@7.2.0) jszip: 3.10.1 transitivePeerDependencies: - debug @@ -7400,15 +7428,15 @@ snapshots: '@types/validate-npm-package-name@4.0.2': {} - '@typescript-eslint/eslint-plugin@8.59.2(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/eslint-plugin@8.59.2(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3)': dependencies: '@eslint-community/regexpp': 4.12.2 - '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) '@typescript-eslint/scope-manager': 8.59.2 - '@typescript-eslint/type-utils': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/type-utils': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) '@typescript-eslint/visitor-keys': 8.59.2 - eslint: 9.39.4(jiti@2.7.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) ignore: 7.0.5 natural-compare: 1.4.0 ts-api-utils: 2.5.0(typescript@6.0.3) @@ -7416,23 +7444,23 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3)': dependencies: '@typescript-eslint/scope-manager': 8.59.2 '@typescript-eslint/types': 8.59.2 - '@typescript-eslint/typescript-estree': 8.59.2(typescript@6.0.3) + '@typescript-eslint/typescript-estree': 8.59.2(supports-color@7.2.0)(typescript@6.0.3) '@typescript-eslint/visitor-keys': 8.59.2 - debug: 4.4.3 - eslint: 9.39.4(jiti@2.7.0) + debug: 4.4.3(supports-color@7.2.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color - '@typescript-eslint/project-service@8.59.2(typescript@6.0.3)': + '@typescript-eslint/project-service@8.59.2(supports-color@7.2.0)(typescript@6.0.3)': dependencies: '@typescript-eslint/tsconfig-utils': 8.59.2(typescript@6.0.3) '@typescript-eslint/types': 8.59.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -7446,13 +7474,13 @@ snapshots: dependencies: typescript: 6.0.3 - '@typescript-eslint/type-utils@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/type-utils@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3)': dependencies: '@typescript-eslint/types': 8.59.2 - '@typescript-eslint/typescript-estree': 8.59.2(typescript@6.0.3) - '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - debug: 4.4.3 - eslint: 9.39.4(jiti@2.7.0) + '@typescript-eslint/typescript-estree': 8.59.2(supports-color@7.2.0)(typescript@6.0.3) + '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + debug: 4.4.3(supports-color@7.2.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) ts-api-utils: 2.5.0(typescript@6.0.3) typescript: 6.0.3 transitivePeerDependencies: @@ -7460,13 +7488,13 @@ snapshots: '@typescript-eslint/types@8.59.2': {} - '@typescript-eslint/typescript-estree@8.59.2(typescript@6.0.3)': + '@typescript-eslint/typescript-estree@8.59.2(supports-color@7.2.0)(typescript@6.0.3)': dependencies: - '@typescript-eslint/project-service': 8.59.2(typescript@6.0.3) + '@typescript-eslint/project-service': 8.59.2(supports-color@7.2.0)(typescript@6.0.3) '@typescript-eslint/tsconfig-utils': 8.59.2(typescript@6.0.3) '@typescript-eslint/types': 8.59.2 '@typescript-eslint/visitor-keys': 8.59.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) minimatch: 10.2.5 semver: 7.7.4 tinyglobby: 0.2.16 @@ -7475,13 +7503,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/utils@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3)': dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.7.0)) + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)) '@typescript-eslint/scope-manager': 8.59.2 '@typescript-eslint/types': 8.59.2 - '@typescript-eslint/typescript-estree': 8.59.2(typescript@6.0.3) - eslint: 9.39.4(jiti@2.7.0) + '@typescript-eslint/typescript-estree': 8.59.2(supports-color@7.2.0)(typescript@6.0.3) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -7635,9 +7663,9 @@ snapshots: acorn@8.16.0: {} - agent-base@6.0.2: + agent-base@6.0.2(supports-color@7.2.0): dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -7784,11 +7812,11 @@ snapshots: axe-core@4.11.4: {} - axios@1.18.1: + axios@1.18.1(debug@4.4.3(supports-color@7.2.0))(supports-color@7.2.0): dependencies: - follow-redirects: 1.16.0 + follow-redirects: 1.16.0(debug@4.4.3(supports-color@7.2.0)) form-data: 4.0.6 - https-proxy-agent: 5.0.1 + https-proxy-agent: 5.0.1(supports-color@7.2.0) proxy-from-env: 2.1.0 transitivePeerDependencies: - debug @@ -7816,11 +7844,11 @@ snapshots: bluebird@3.4.7: {} - body-parser@2.2.2: + body-parser@2.2.2(supports-color@7.2.0): dependencies: bytes: 3.1.2 content-type: 1.0.5 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) http-errors: 2.0.1 iconv-lite: 0.7.2 on-finished: 2.4.1 @@ -8021,13 +8049,17 @@ snapshots: es-errors: 1.3.0 is-data-view: 1.0.2 - debug@3.2.7: + debug@3.2.7(supports-color@7.2.0): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 7.2.0 - debug@4.4.3: + debug@4.4.3(supports-color@7.2.0): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 7.2.0 decimal.js@10.6.0: {} @@ -8350,18 +8382,18 @@ snapshots: escape-string-regexp@5.0.0: {} - eslint-config-next@16.2.4(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3): + eslint-config-next@16.2.4(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3): dependencies: '@next/eslint-plugin-next': 16.2.4 - eslint: 9.39.4(jiti@2.7.0) - eslint-import-resolver-node: 0.3.10 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)) - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)) - eslint-plugin-jsx-a11y: 6.10.2(eslint@9.39.4(jiti@2.7.0)) - eslint-plugin-react: 7.37.5(eslint@9.39.4(jiti@2.7.0)) - eslint-plugin-react-hooks: 7.1.1(eslint@9.39.4(jiti@2.7.0)) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) + eslint-import-resolver-node: 0.3.10(supports-color@7.2.0) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) + eslint-plugin-jsx-a11y: 6.10.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)) + eslint-plugin-react: 7.37.5(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)) + eslint-plugin-react-hooks: 7.1.1(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) globals: 16.4.0 - typescript-eslint: 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) + typescript-eslint: 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) optionalDependencies: typescript: 6.0.3 transitivePeerDependencies: @@ -8370,52 +8402,52 @@ snapshots: - eslint-plugin-import-x - supports-color - eslint-import-resolver-node@0.3.10: + eslint-import-resolver-node@0.3.10(supports-color@7.2.0): dependencies: - debug: 3.2.7 + debug: 3.2.7(supports-color@7.2.0) is-core-module: 2.16.2 resolve: 2.0.0-next.6 transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)): + eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0): dependencies: '@nolyfill/is-core-module': 1.0.39 - debug: 4.4.3 - eslint: 9.39.4(jiti@2.7.0) + debug: 4.4.3(supports-color@7.2.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) get-tsconfig: 4.14.0 is-bun-module: 2.0.0 stable-hash: 0.0.5 tinyglobby: 0.2.16 unrs-resolver: 1.11.1 optionalDependencies: - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.1(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-node@0.3.10)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)): + eslint-module-utils@2.12.1(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint-import-resolver-node@0.3.10(supports-color@7.2.0))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0): dependencies: - debug: 3.2.7 + debug: 3.2.7(supports-color@7.2.0) optionalDependencies: - '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - eslint: 9.39.4(jiti@2.7.0) - eslint-import-resolver-node: 0.3.10 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)) + '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) + eslint-import-resolver-node: 0.3.10(supports-color@7.2.0) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) transitivePeerDependencies: - supports-color - eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)): + eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0): dependencies: '@rtsao/scc': 1.1.0 array-includes: 3.1.9 array.prototype.findlastindex: 1.2.6 array.prototype.flat: 1.3.3 array.prototype.flatmap: 1.3.3 - debug: 3.2.7 + debug: 3.2.7(supports-color@7.2.0) doctrine: 2.1.0 - eslint: 9.39.4(jiti@2.7.0) - eslint-import-resolver-node: 0.3.10 - eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-node@0.3.10)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) + eslint-import-resolver-node: 0.3.10(supports-color@7.2.0) + eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint-import-resolver-node@0.3.10(supports-color@7.2.0))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) hasown: 2.0.3 is-core-module: 2.16.2 is-glob: 4.0.3 @@ -8427,13 +8459,13 @@ snapshots: string.prototype.trimend: 1.0.9 tsconfig-paths: 3.15.0 optionalDependencies: - '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) transitivePeerDependencies: - eslint-import-resolver-typescript - eslint-import-resolver-webpack - supports-color - eslint-plugin-jsx-a11y@6.10.2(eslint@9.39.4(jiti@2.7.0)): + eslint-plugin-jsx-a11y@6.10.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)): dependencies: aria-query: 5.3.2 array-includes: 3.1.9 @@ -8443,7 +8475,7 @@ snapshots: axobject-query: 4.1.0 damerau-levenshtein: 1.0.8 emoji-regex: 9.2.2 - eslint: 9.39.4(jiti@2.7.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) hasown: 2.0.3 jsx-ast-utils: 3.3.5 language-tags: 1.0.9 @@ -8452,18 +8484,18 @@ snapshots: safe-regex-test: 1.1.0 string.prototype.includes: 2.0.1 - eslint-plugin-react-hooks@7.1.1(eslint@9.39.4(jiti@2.7.0)): + eslint-plugin-react-hooks@7.1.1(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0): dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/parser': 7.29.3 - eslint: 9.39.4(jiti@2.7.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) hermes-parser: 0.25.1 zod: 4.4.3 zod-validation-error: 4.0.2(zod@4.4.3) transitivePeerDependencies: - supports-color - eslint-plugin-react@7.37.5(eslint@9.39.4(jiti@2.7.0)): + eslint-plugin-react@7.37.5(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)): dependencies: array-includes: 3.1.9 array.prototype.findlast: 1.2.5 @@ -8471,7 +8503,7 @@ snapshots: array.prototype.tosorted: 1.1.4 doctrine: 2.1.0 es-iterator-helpers: 1.3.2 - eslint: 9.39.4(jiti@2.7.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) estraverse: 5.3.0 hasown: 2.0.3 jsx-ast-utils: 3.3.5 @@ -8496,14 +8528,14 @@ snapshots: eslint-visitor-keys@5.0.1: {} - eslint@9.39.4(jiti@2.7.0): + eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0): dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.7.0)) + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)) '@eslint-community/regexpp': 4.12.2 - '@eslint/config-array': 0.21.2 + '@eslint/config-array': 0.21.2(supports-color@7.2.0) '@eslint/config-helpers': 0.4.2 '@eslint/core': 0.17.0 - '@eslint/eslintrc': 3.3.5 + '@eslint/eslintrc': 3.3.5(supports-color@7.2.0) '@eslint/js': 9.39.4 '@eslint/plugin-kit': 0.4.1 '@humanfs/node': 0.16.8 @@ -8513,7 +8545,7 @@ snapshots: ajv: 6.15.0 chalk: 4.1.2 cross-spawn: 7.0.6 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) escape-string-regexp: 4.0.0 eslint-scope: 8.4.0 eslint-visitor-keys: 4.2.1 @@ -8600,25 +8632,25 @@ snapshots: expect-type@1.3.0: {} - express-rate-limit@8.5.0(express@5.2.1): + express-rate-limit@8.5.0(express@5.2.1(supports-color@7.2.0)): dependencies: - express: 5.2.1 + express: 5.2.1(supports-color@7.2.0) ip-address: 10.1.0 - express@5.2.1: + express@5.2.1(supports-color@7.2.0): dependencies: accepts: 2.0.0 - body-parser: 2.2.2 + body-parser: 2.2.2(supports-color@7.2.0) content-disposition: 1.1.0 content-type: 1.0.5 cookie: 0.7.2 cookie-signature: 1.2.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) depd: 2.0.0 encodeurl: 2.0.0 escape-html: 1.0.3 etag: 1.8.1 - finalhandler: 2.1.1 + finalhandler: 2.1.1(supports-color@7.2.0) fresh: 2.0.0 http-errors: 2.0.1 merge-descriptors: 2.0.0 @@ -8629,9 +8661,9 @@ snapshots: proxy-addr: 2.0.7 qs: 6.15.1 range-parser: 1.2.1 - router: 2.2.0 - send: 1.2.1 - serve-static: 2.2.1 + router: 2.2.0(supports-color@7.2.0) + send: 1.2.1(supports-color@7.2.0) + serve-static: 2.2.1(supports-color@7.2.0) statuses: 2.0.2 type-is: 2.0.1 vary: 1.1.2 @@ -8709,9 +8741,9 @@ snapshots: dependencies: to-regex-range: 5.0.1 - finalhandler@2.1.1: + finalhandler@2.1.1(supports-color@7.2.0): dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) encodeurl: 2.0.0 escape-html: 1.0.3 on-finished: 2.4.1 @@ -8734,7 +8766,9 @@ snapshots: flatted@3.4.2: {} - follow-redirects@1.16.0: {} + follow-redirects@1.16.0(debug@4.4.3(supports-color@7.2.0)): + optionalDependencies: + debug: 4.4.3(supports-color@7.2.0) for-each@0.3.5: dependencies: @@ -8886,7 +8920,7 @@ snapshots: dependencies: function-bind: 1.1.2 - hast-util-to-jsx-runtime@2.3.6: + hast-util-to-jsx-runtime@2.3.6(supports-color@7.2.0): dependencies: '@types/estree': 1.0.8 '@types/hast': 3.0.4 @@ -8895,9 +8929,9 @@ snapshots: devlop: 1.1.0 estree-util-is-identifier-name: 3.0.0 hast-util-whitespace: 3.0.0 - mdast-util-mdx-expression: 2.0.1 - mdast-util-mdx-jsx: 3.2.0 - mdast-util-mdxjs-esm: 2.0.1 + mdast-util-mdx-expression: 2.0.1(supports-color@7.2.0) + mdast-util-mdx-jsx: 3.2.0(supports-color@7.2.0) + mdast-util-mdxjs-esm: 2.0.1(supports-color@7.2.0) property-information: 7.1.0 space-separated-tokens: 2.0.2 style-to-js: 1.1.21 @@ -8939,17 +8973,17 @@ snapshots: statuses: 2.0.2 toidentifier: 1.0.1 - https-proxy-agent@5.0.1: + https-proxy-agent@5.0.1(supports-color@7.2.0): dependencies: - agent-base: 6.0.2 - debug: 4.4.3 + agent-base: 6.0.2(supports-color@7.2.0) + debug: 4.4.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color - https-proxy-agent@7.0.6: + https-proxy-agent@7.0.6(supports-color@7.2.0): dependencies: agent-base: 7.1.4 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -9398,14 +9432,14 @@ snapshots: unist-util-is: 6.0.1 unist-util-visit-parents: 6.0.2 - mdast-util-from-markdown@2.0.3: + mdast-util-from-markdown@2.0.3(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 '@types/unist': 3.0.3 decode-named-character-reference: 1.3.0 devlop: 1.1.0 mdast-util-to-string: 4.0.0 - micromark: 4.0.2 + micromark: 4.0.2(supports-color@7.2.0) micromark-util-decode-numeric-character-reference: 2.0.2 micromark-util-decode-string: 2.0.1 micromark-util-normalize-identifier: 2.0.1 @@ -9423,67 +9457,67 @@ snapshots: mdast-util-find-and-replace: 3.0.2 micromark-util-character: 2.1.1 - mdast-util-gfm-footnote@2.1.0: + mdast-util-gfm-footnote@2.1.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 micromark-util-normalize-identifier: 2.0.1 transitivePeerDependencies: - supports-color - mdast-util-gfm-strikethrough@2.0.0: + mdast-util-gfm-strikethrough@2.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-gfm-table@2.0.0: + mdast-util-gfm-table@2.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 devlop: 1.1.0 markdown-table: 3.0.4 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-gfm-task-list-item@2.0.0: + mdast-util-gfm-task-list-item@2.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-gfm@3.1.0: + mdast-util-gfm@3.1.0(supports-color@7.2.0): dependencies: - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-gfm-autolink-literal: 2.0.1 - mdast-util-gfm-footnote: 2.1.0 - mdast-util-gfm-strikethrough: 2.0.0 - mdast-util-gfm-table: 2.0.0 - mdast-util-gfm-task-list-item: 2.0.0 + mdast-util-gfm-footnote: 2.1.0(supports-color@7.2.0) + mdast-util-gfm-strikethrough: 2.0.0(supports-color@7.2.0) + mdast-util-gfm-table: 2.0.0(supports-color@7.2.0) + mdast-util-gfm-task-list-item: 2.0.0(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-mdx-expression@2.0.1: + mdast-util-mdx-expression@2.0.1(supports-color@7.2.0): dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-mdx-jsx@3.2.0: + mdast-util-mdx-jsx@3.2.0(supports-color@7.2.0): dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 @@ -9491,7 +9525,7 @@ snapshots: '@types/unist': 3.0.3 ccount: 2.0.1 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 parse-entities: 4.0.2 stringify-entities: 4.0.4 @@ -9500,13 +9534,13 @@ snapshots: transitivePeerDependencies: - supports-color - mdast-util-mdxjs-esm@2.0.1: + mdast-util-mdxjs-esm@2.0.1(supports-color@7.2.0): dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color @@ -9727,10 +9761,10 @@ snapshots: micromark-util-types@2.0.2: {} - micromark@4.0.2: + micromark@4.0.2(supports-color@7.2.0): dependencies: '@types/debug': 4.1.13 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) decode-named-character-reference: 1.3.0 devlop: 1.1.0 micromark-core-commonmark: 2.0.3 @@ -9842,7 +9876,7 @@ snapshots: react: 19.2.4 react-dom: 19.2.4(react@19.2.4) - next@16.2.4(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + next@16.2.4(@babel/core@7.29.0(supports-color@7.2.0))(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4): dependencies: '@next/env': 16.2.4 '@swc/helpers': 0.5.15 @@ -9851,7 +9885,7 @@ snapshots: postcss: 8.4.31 react: 19.2.4 react-dom: 19.2.4(react@19.2.4) - styled-jsx: 5.1.6(@babel/core@7.29.0)(react@19.2.4) + styled-jsx: 5.1.6(@babel/core@7.29.0(supports-color@7.2.0))(react@19.2.4) optionalDependencies: '@next/swc-darwin-arm64': 16.2.4 '@next/swc-darwin-x64': 16.2.4 @@ -10183,17 +10217,17 @@ snapshots: react-is@17.0.2: {} - react-markdown@10.1.0(@types/react@19.2.14)(react@19.2.4): + react-markdown@10.1.0(@types/react@19.2.14)(react@19.2.4)(supports-color@7.2.0): dependencies: '@types/hast': 3.0.4 '@types/mdast': 4.0.4 '@types/react': 19.2.14 devlop: 1.1.0 - hast-util-to-jsx-runtime: 2.3.6 + hast-util-to-jsx-runtime: 2.3.6(supports-color@7.2.0) html-url-attributes: 3.0.1 mdast-util-to-hast: 13.2.1 react: 19.2.4 - remark-parse: 11.0.0 + remark-parse: 11.0.0(supports-color@7.2.0) remark-rehype: 11.1.2 unified: 11.0.5 unist-util-visit: 5.1.0 @@ -10283,21 +10317,21 @@ snapshots: gopd: 1.2.0 set-function-name: 2.0.2 - remark-gfm@4.0.1: + remark-gfm@4.0.1(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 - mdast-util-gfm: 3.1.0 + mdast-util-gfm: 3.1.0(supports-color@7.2.0) micromark-extension-gfm: 3.0.0 - remark-parse: 11.0.0 + remark-parse: 11.0.0(supports-color@7.2.0) remark-stringify: 11.0.0 unified: 11.0.5 transitivePeerDependencies: - supports-color - remark-parse@11.0.0: + remark-parse@11.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) micromark-util-types: 2.0.2 unified: 11.0.5 transitivePeerDependencies: @@ -10366,9 +10400,9 @@ snapshots: '@rolldown/binding-win32-arm64-msvc': 1.0.0-rc.17 '@rolldown/binding-win32-x64-msvc': 1.0.0-rc.17 - router@2.2.0: + router@2.2.0(supports-color@7.2.0): dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) depd: 2.0.0 is-promise: 4.0.0 parseurl: 1.3.3 @@ -10420,9 +10454,9 @@ snapshots: semver@7.7.4: {} - send@1.2.1: + send@1.2.1(supports-color@7.2.0): dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) encodeurl: 2.0.0 escape-html: 1.0.3 etag: 1.8.1 @@ -10436,12 +10470,12 @@ snapshots: transitivePeerDependencies: - supports-color - serve-static@2.2.1: + serve-static@2.2.1(supports-color@7.2.0): dependencies: encodeurl: 2.0.0 escape-html: 1.0.3 parseurl: 1.3.3 - send: 1.2.1 + send: 1.2.1(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -10475,14 +10509,14 @@ snapshots: setprototypeof@1.2.0: {} - shadcn@4.7.0(@types/node@20.19.39)(typescript@6.0.3): + shadcn@4.7.0(@types/node@20.19.39)(supports-color@7.2.0)(typescript@6.0.3): dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/parser': 7.29.3 - '@babel/plugin-transform-typescript': 7.28.6(@babel/core@7.29.0) - '@babel/preset-typescript': 7.28.5(@babel/core@7.29.0) + '@babel/plugin-transform-typescript': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/preset-typescript': 7.28.5(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) '@dotenvx/dotenvx': 1.65.0 - '@modelcontextprotocol/sdk': 1.29.0(zod@3.25.76) + '@modelcontextprotocol/sdk': 1.30.0(supports-color@7.2.0)(zod@3.25.76) '@types/validate-npm-package-name': 4.0.2 browserslist: 4.28.2 commander: 14.0.3 @@ -10494,7 +10528,7 @@ snapshots: fast-glob: 3.3.3 fs-extra: 11.3.4 fuzzysort: 3.1.0 - https-proxy-agent: 7.0.6 + https-proxy-agent: 7.0.6(supports-color@7.2.0) kleur: 4.1.5 msw: 2.14.3(@types/node@20.19.39)(typescript@6.0.3) node-fetch: 3.3.2 @@ -10725,12 +10759,12 @@ snapshots: dependencies: inline-style-parser: 0.2.7 - styled-jsx@5.1.6(@babel/core@7.29.0)(react@19.2.4): + styled-jsx@5.1.6(@babel/core@7.29.0(supports-color@7.2.0))(react@19.2.4): dependencies: client-only: 0.0.1 react: 19.2.4 optionalDependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) supports-color@7.2.0: dependencies: @@ -10873,13 +10907,13 @@ snapshots: possible-typed-array-names: 1.1.0 reflect.getprototypeof: 1.0.10 - typescript-eslint@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3): + typescript-eslint@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3): dependencies: - '@typescript-eslint/eslint-plugin': 8.59.2(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - '@typescript-eslint/typescript-estree': 8.59.2(typescript@6.0.3) - '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - eslint: 9.39.4(jiti@2.7.0) + '@typescript-eslint/eslint-plugin': 8.59.2(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + '@typescript-eslint/typescript-estree': 8.59.2(supports-color@7.2.0)(typescript@6.0.3) + '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -11185,6 +11219,10 @@ snapshots: dependencies: zod: 3.25.76 + zod-to-json-schema@3.25.2(zod@4.4.3): + dependencies: + zod: 4.4.3 + zod-validation-error@4.0.2(zod@4.4.3): dependencies: zod: 4.4.3 diff --git a/scripts/guanxin-case/constants.ts b/scripts/guanxin-case/constants.ts deleted file mode 100644 index d4f60992..00000000 --- a/scripts/guanxin-case/constants.ts +++ /dev/null @@ -1,8 +0,0 @@ -/** - * Dedicated Notebook workspace for the 观心 cardiovascular case. - * workspaces 表没有 title 列，用稳定 userId 标识。 - */ -export const GUANXIN_CASE_USER_ID = "case:guanxin-cardiovascular" - -/** 方案约定的 pilot 规模（20–30）按 sheet×难度 6 层均分。 */ -export const PILOT_CASES_PER_STRATUM = 4 diff --git a/scripts/guanxin-case/ensure-workspace.mts b/scripts/guanxin-case/ensure-workspace.mts deleted file mode 100644 index 963bca61..00000000 --- a/scripts/guanxin-case/ensure-workspace.mts +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Ensure the dedicated 观心 case workspace exists in the Notebook database. - * Requires DATABASE_URL (Notebook Neon/Postgres), not the Knowhere eval DSN. - * - * DATABASE_URL=... node --experimental-strip-types scripts/guanxin-case/ensure-workspace.mts - */ -import { workspaceService } from "../../src/domains/workspace/service.ts" -import { GUANXIN_CASE_USER_ID } from "./constants.ts" - -const workspace = await workspaceService.ensureWorkspace(GUANXIN_CASE_USER_ID) -console.log( - JSON.stringify( - { - userId: workspace.userId, - workspaceId: workspace.id, - namespace: workspace.namespace, - }, - null, - 2, - ), -) diff --git a/scripts/guanxin-case/sample-pilot.py b/scripts/guanxin-case/sample-pilot.py deleted file mode 100644 index a743d4da..00000000 --- a/scripts/guanxin-case/sample-pilot.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -"""Stratified pilot sample from 观心 knowhere自测集.xlsx. - -Takes the first PILOT_CASES_PER_STRATUM rows (by seq_id) from each -(sheet, 难度) bucket. Does not call Knowhere or write the case workspace. -""" -from __future__ import annotations - -import json -import sys -from collections import defaultdict -from pathlib import Path - -import openpyxl - -ROOT = Path(__file__).resolve().parents[2] -DEFAULT_XLSX = Path( - "/Users/wuchengke/Desktop/skills-coding/观心2.0-RAG-v1.1-demo/knowhere自测集.xlsx" -) -DEFAULT_OUT = ROOT / ".tmp" / "guanxin-pilot-cases.json" -PILOT_CASES_PER_STRATUM = 4 - - -def load_rows(xlsx: Path) -> list[dict]: - wb = openpyxl.load_workbook(xlsx, read_only=True, data_only=True) - rows: list[dict] = [] - try: - for sheet_name in wb.sheetnames: - sheet_rows = list(wb[sheet_name].iter_rows(values_only=True)) - if not sheet_rows: - continue - header = [ - str(h).strip() if h is not None else f"col{i}" - for i, h in enumerate(sheet_rows[0]) - ] - for raw in sheet_rows[1:]: - if not raw or raw[0] in (None, ""): - continue - item = { - header[i]: (raw[i] if i < len(raw) else None) - for i in range(len(header)) - } - query = str(item.get("具体query") or "").strip() - if not query: - continue - rows.append( - { - "sheet": sheet_name, - "seq_id": str(item.get("seq_id") or ""), - "query": query, - "disease": str(item.get("具体疾病名称") or "").strip(), - "scene": str(item.get("应用场景-考察能力") or "").strip(), - "difficulty": str(item.get("难度") or "").strip(), - "input_type": str(item.get("输入类型") or "").strip(), - } - ) - finally: - wb.close() - return rows - - -def sample(rows: list[dict]) -> list[dict]: - buckets: dict[tuple[str, str], list[dict]] = defaultdict(list) - for row in rows: - buckets[(row["sheet"], row["difficulty"] or "?")].append(row) - picked: list[dict] = [] - for key in sorted(buckets): - group = sorted(buckets[key], key=lambda row: row["seq_id"]) - picked.extend(group[:PILOT_CASES_PER_STRATUM]) - return sorted(picked, key=lambda row: (row["sheet"], row["seq_id"])) - - -def main() -> None: - xlsx = Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_XLSX - out = Path(sys.argv[2]) if len(sys.argv) > 2 else DEFAULT_OUT - rows = load_rows(xlsx) - picked = sample(rows) - out.parent.mkdir(parents=True, exist_ok=True) - payload = { - "xlsx": str(xlsx), - "per_stratum": PILOT_CASES_PER_STRATUM, - "source_count": len(rows), - "pilot_count": len(picked), - "cases": picked, - } - out.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - print(f"wrote {len(picked)} / {len(rows)} -> {out}") - - -if __name__ == "__main__": - main() diff --git a/src/agent-harness/runtime.test.ts b/src/agent-harness/runtime.test.ts index c2066c3d..b459d5b8 100644 --- a/src/agent-harness/runtime.test.ts +++ b/src/agent-harness/runtime.test.ts @@ -655,7 +655,6 @@ describe("agent harness runtime", () => { const state: { finalizedManifest?: OutputManifest finalized?: boolean - memorySearchInvoked?: boolean } = {} const tools = createHarnessTools({ state, @@ -671,7 +670,6 @@ describe("agent harness runtime", () => { expect(searchText).toContain('') expect(searchText).toContain('ref="mem:1"') expect(searchText).toContain('itemId="item_1"') - expect(state.memorySearchInvoked).toBe(true) expect(search).toHaveBeenCalledWith({ query: "毛利率", kinds: undefined, @@ -1008,10 +1006,9 @@ describe("agent harness runtime", () => { expect(result.activeTools).not.toContain("knowhere_search") }) - it("keeps Knowhere tools closed for no_retrieval even after memory_search", () => { + it("keeps Knowhere tools closed for no_retrieval", () => { const result = prepareHarnessStep({ stepNumber: 4, - memorySearchInvoked: true, intent: { task: "answer", dependsOnPreviousTurn: false, @@ -1027,8 +1024,8 @@ describe("agent harness runtime", () => { expect(result.activeTools).not.toContain("knowhere_search") }) - it("opens Knowhere tools only after memory_search when sources are required", () => { - const beforeMemory = prepareHarnessStep({ + it("opens memory_search and knowhere_search together as peers when sources are required", () => { + const result = prepareHarnessStep({ stepNumber: 3, intent: { task: "answer", @@ -1040,23 +1037,8 @@ describe("agent harness runtime", () => { }, messages: [], }) - const afterMemory = prepareHarnessStep({ - stepNumber: 4, - memorySearchInvoked: true, - intent: { - task: "answer", - dependsOnPreviousTurn: false, - retrievalNeeded: "yes", - targetModalities: ["text"], - constraints: {}, - groundingPolicy: "must_use_sources", - }, - messages: [], - }) - expect(beforeMemory.activeTools).toContain("memory_search") - expect(beforeMemory.activeTools).not.toContain("knowhere_search") - expect(afterMemory.activeTools).toEqual( + expect(result.activeTools).toEqual( expect.arrayContaining([ "memory_search", "knowhere_search", diff --git a/src/agent-harness/runtime.ts b/src/agent-harness/runtime.ts index 11a6c3aa..0f4aab0a 100644 --- a/src/agent-harness/runtime.ts +++ b/src/agent-harness/runtime.ts @@ -60,7 +60,6 @@ type HarnessToolState = { inspectedImageRefs?: string[] imageHighlights?: ImageInspectionHighlights[] toolCalls?: HarnessToolCallTrace[] - memorySearchInvoked?: boolean } type HarnessTools = ReturnType @@ -245,7 +244,6 @@ export async function runAgentHarness( messages: stepMessages, stepNumber, intent: state.intent, - memorySearchInvoked: state.memorySearchInvoked === true, hasUninspectedImageAssets: input.inspectImages !== undefined && hasUninspectedImageAssets({ state, ledger }), @@ -299,12 +297,21 @@ const crystalRetrievalTools = [ /** Reserved third retrieval slot (cognition). Not registered this round. */ const cognitionRetrievalTools = [] as const +// TODO(memory-architecture): today the agent itself decides, per turn via +// declareIntent, whether to call memory_search / knowhere_search as MCP +// tools. An alternative considered and deferred: always query Memento +// (including future "cognition") on every turn and let Memento decide what, +// if anything, to inject into context, instead of the agent choosing to call +// a tool. Not adopted now — it would replace this tool-invocation control +// flow with a middleware/auto-inject model and needs its own design + test +// rewrite. Revisit if agent misjudgment on retrieval-needed becomes a real +// problem. + export function prepareHarnessStep(input: { readonly stepNumber: number readonly messages: readonly ModelMessage[] readonly hasUninspectedImageAssets?: boolean readonly intent?: IntentFrame - readonly memorySearchInvoked?: boolean }): HarnessStepPreparation { const messages = sanitizeHarnessModelMessagesForStep(input.messages) @@ -350,14 +357,12 @@ export function prepareHarnessStep(input: { messages, activeTools: selectHarnessActiveTools({ intent: input.intent, - memorySearchInvoked: input.memorySearchInvoked === true, }), } } function selectHarnessActiveTools(input: { readonly intent?: IntentFrame - readonly memorySearchInvoked: boolean }): Array> { const tools: Array> = [ ...alwaysAvailableTools, @@ -366,11 +371,11 @@ function selectHarnessActiveTools(input: { return tools } + // memory_search and knowhere_search are peers: both open together once + // retrieval is allowed. The agent decides which to call and in what + // order — neither tool gates the other. tools.push(...fluidRetrievalTools) - if ( - input.intent?.groundingPolicy === "must_use_sources" && - input.memorySearchInvoked - ) { + if (input.intent?.groundingPolicy === "must_use_sources") { tools.push(...crystalRetrievalTools) } tools.push(...cognitionRetrievalTools) @@ -535,12 +540,10 @@ export function createHarnessTools(input: { toolName: "memory_search", inputSummary: summarizeMemorySearchRequest(request), execute: async () => { - const output = await executeMemorySearch({ + return await executeMemorySearch({ memoryTools: input.memoryTools, request, }) - input.state.memorySearchInvoked = true - return output }, summarizeOutput: summarizeMemoryTextOutput, }), diff --git a/src/app/api/memory/distill/route.ts b/src/app/api/memory/distill/route.ts deleted file mode 100644 index e0230d93..00000000 --- a/src/app/api/memory/distill/route.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { serve } from "@upstash/workflow/nextjs" - -import { - normalizeMemoryDistillPayload, - runMemoryDistillWorkflow, - type MemoryDistillPayload, -} from "@/domains/memory/distill-workflow" -import { logger } from "@/lib/logger" - -export const { POST } = serve( - async (context) => { - const payload = normalizeMemoryDistillPayload(context.requestPayload) - if (!payload) { - logger.warn("memory: distill workflow received invalid payload") - return - } - await runMemoryDistillWorkflow({ context, payload }) - }, - { - failureFunction: async ({ context, failResponse }) => { - logger.error("memory: distill workflow failed", { - payload: context.requestPayload, - failResponse, - }) - }, - }, -) diff --git a/src/app/api/memory/extract/route.ts b/src/app/api/memory/extract/route.ts deleted file mode 100644 index ee25dbe4..00000000 --- a/src/app/api/memory/extract/route.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { serve } from "@upstash/workflow/nextjs" - -import { - normalizeMemoryExtractPayload, - runMemoryExtractWorkflow, - type MemoryExtractPayload, -} from "@/domains/memory/extract-workflow" -import { logger } from "@/lib/logger" - -export const { POST } = serve( - async (context) => { - const payload = normalizeMemoryExtractPayload(context.requestPayload) - if (!payload) { - logger.warn("memory: extract workflow received invalid payload") - return - } - await runMemoryExtractWorkflow({ context, payload }) - }, - { - failureFunction: async ({ context, failResponse }) => { - logger.error("memory: extract workflow failed", { - payload: context.requestPayload, - failResponse, - }) - }, - }, -) diff --git a/src/domains/chat/commit-turn.test.ts b/src/domains/chat/commit-turn.test.ts index 62e20b31..97f19dae 100644 --- a/src/domains/chat/commit-turn.test.ts +++ b/src/domains/chat/commit-turn.test.ts @@ -3,7 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest" const mocks = vi.hoisted(() => ({ handleChatTurn: vi.fn(), - triggerMemoryExtraction: vi.fn(), + captureMemoryTurn: vi.fn(), recordActivations: vi.fn(), })) @@ -11,14 +11,9 @@ vi.mock("./service", () => ({ handleChatTurn: mocks.handleChatTurn, })) -vi.mock("@/domains/memory/extract-trigger", () => ({ - triggerMemoryExtraction: mocks.triggerMemoryExtraction, -})) - -vi.mock("@/domains/retrieval-activation/service", () => ({ - retrievalActivationService: { - recordActivations: mocks.recordActivations, - }, +vi.mock("@/integrations/memento/client", () => ({ + captureMemoryTurn: mocks.captureMemoryTurn, + recordActivations: mocks.recordActivations, })) import { commitChatTurn } from "./commit-turn" @@ -27,7 +22,8 @@ import type { Workspace } from "@/infrastructure/db/schema" describe("commitChatTurn", () => { beforeEach(() => { vi.clearAllMocks() - mocks.recordActivations.mockResolvedValue(1) + mocks.recordActivations.mockResolvedValue(undefined) + mocks.captureMemoryTurn.mockResolvedValue(undefined) }) it("records fluid memory activations from finalize after a successful turn", async () => { @@ -105,11 +101,12 @@ describe("commitChatTurn", () => { }) expect(Either.isRight(result)).toBe(true) - expect(mocks.triggerMemoryExtraction).toHaveBeenCalledWith({ + expect(mocks.captureMemoryTurn).toHaveBeenCalledWith({ workspaceId: "workspace_1", - threadId: "thread_1", - userMessageId: "msg_user", - assistantMessageId: "msg_assistant", + sourceMessageId: "msg_assistant", + userText: "毛利率", + assistantText: "按已有记忆。", + referencedDocumentIds: ["doc_1"], }) expect(mocks.recordActivations).toHaveBeenCalledWith([ { diff --git a/src/domains/chat/commit-turn.ts b/src/domains/chat/commit-turn.ts index 373bb3a6..1113058b 100644 --- a/src/domains/chat/commit-turn.ts +++ b/src/domains/chat/commit-turn.ts @@ -1,12 +1,11 @@ import { Either } from "effect" import type { MemoryCitation } from "@/agent-harness" +import { + captureMemoryTurn, + recordActivations, +} from "@/integrations/memento/client" import { generateAgenticOutputManifest } from "./prompt" -import { triggerMemoryExtraction } from "@/domains/memory/extract-trigger" -import { retrievalActivationService } from "@/domains/retrieval-activation/service" -import { toChunkUnitRef } from "@/domains/retrieval-activation/types" -import { summarizeUnknownError } from "@/lib/format-log-value" -import { logger } from "@/lib/logger" import type { ChatCitationView } from "./types" import { handleChatTurn, @@ -17,9 +16,8 @@ import { type CommitChatTurnInput = Parameters[0] /** - * Production chat-turn commit used by the HTTP route and the 观心 batch - * script: answer → persist → extract fluid memory → record cited - * crystal/memory activations. + * Production chat-turn commit used by the HTTP route: answer → persist → + * capture fluid memory on Memento → record cited crystal/memory activations. */ export async function commitChatTurn( input: CommitChatTurnInput, @@ -37,15 +35,19 @@ export async function commitChatTurn( }) if (Either.isRight(result)) { - void triggerMemoryExtraction({ + const [userMessage, assistantMessage] = result.right.messages + void captureMemoryTurn({ workspaceId: input.workspace.id, - threadId: result.right.threadId, - userMessageId: result.right.messages[0].id, - assistantMessageId: result.right.messages[1].id, + sourceMessageId: assistantMessage.id, + userText: userMessage.content, + assistantText: assistantMessage.content, + referencedDocumentIds: collectCitationDocumentIds( + assistantMessage.citations, + ), }) void recordChunkActivations({ workspaceId: input.workspace.id, - citations: result.right.messages[1].citations, + citations: assistantMessage.citations, }) void recordMemoryActivations({ workspaceId: input.workspace.id, @@ -91,17 +93,7 @@ export async function recordChunkActivations(input: { }, ] }) - if (activationInputs.length === 0) return - - try { - await retrievalActivationService.recordActivations(activationInputs) - } catch (error) { - logger.warn("chat: failed to record chunk activations", { - workspaceId: input.workspaceId, - chunkCount: activationInputs.length, - error: summarizeUnknownError(error), - }) - } + await recordActivations(activationInputs) } /** @@ -117,15 +109,25 @@ export async function recordMemoryActivations(input: { unitType: "fluid_memory" as const, unitRef: citation.itemId, })) - if (activationInputs.length === 0) return + await recordActivations(activationInputs) +} - try { - await retrievalActivationService.recordActivations(activationInputs) - } catch (error) { - logger.warn("chat: failed to record memory activations", { - workspaceId: input.workspaceId, - memoryCount: activationInputs.length, - error: summarizeUnknownError(error), - }) +function toChunkUnitRef(input: { + readonly documentId: string + readonly chunkId: string +}): string { + return `${input.documentId}:${input.chunkId}` +} + +function collectCitationDocumentIds( + citations: readonly ChatCitationView[] | undefined, +): string[] { + const ids = new Set() + for (const citation of citations ?? []) { + const documentId = citation.source.documentId + if (typeof documentId === "string" && documentId.length > 0) { + ids.add(documentId) + } } + return [...ids] } diff --git a/src/domains/chat/memory-tools.test.ts b/src/domains/chat/memory-tools.test.ts deleted file mode 100644 index ad083a90..00000000 --- a/src/domains/chat/memory-tools.test.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from "vitest" - -import { DISTILL_DEDUP_CANDIDATES_PER_KIND } from "@/domains/memory/distill-config" -import type { FluidMemoryItem } from "@/infrastructure/db/schema" - -const findDedupCandidates = vi.fn() - -vi.mock("@/domains/memory/service", () => ({ - memoryService: { - findDedupCandidates: (...args: unknown[]) => findDedupCandidates(...args), - }, -})) - -describe("notebookMemoryTools", () => { - beforeEach(() => { - findDedupCandidates.mockReset() - }) - - it("queries all four kinds and assigns mem refs in kind order", async () => { - findDedupCandidates.mockImplementation( - async (_workspaceId: string, kind: string) => { - if (kind === "stance") return [makeMemoryItem({ id: "item_stance" })] - if (kind === "entity_of_interest") { - return [makeMemoryItem({ id: "item_entity", kind: "entity_of_interest" })] - } - return [] - }, - ) - const { notebookMemoryTools } = await import("./memory-tools") - const runtime = notebookMemoryTools.createRuntime({ - workspaceId: "workspace_1", - }) - - const response = await runtime.search({ query: "毛利率 英伟达" }) - - expect(findDedupCandidates).toHaveBeenCalledTimes(4) - expect(findDedupCandidates.mock.calls.map((call) => call[1])).toEqual([ - "indicator_pref", - "stance", - "decision_rule", - "entity_of_interest", - ]) - expect(findDedupCandidates.mock.calls[0]?.[3]).toBe( - DISTILL_DEDUP_CANDIDATES_PER_KIND, - ) - expect(response).toEqual({ - query: "毛利率 英伟达", - items: [ - expect.objectContaining({ - ref: "mem:1", - itemId: "item_stance", - kind: "stance", - }), - expect.objectContaining({ - ref: "mem:2", - itemId: "item_entity", - kind: "entity_of_interest", - }), - ], - }) - }) - - it("searches only the requested kinds", async () => { - findDedupCandidates.mockResolvedValue([]) - const { notebookMemoryTools } = await import("./memory-tools") - const runtime = notebookMemoryTools.createRuntime({ - workspaceId: "workspace_1", - }) - - await runtime.search({ - query: "PE", - kinds: ["indicator_pref"], - }) - - expect(findDedupCandidates).toHaveBeenCalledTimes(1) - expect(findDedupCandidates).toHaveBeenCalledWith( - "workspace_1", - "indicator_pref", - expect.any(Array), - DISTILL_DEDUP_CANDIDATES_PER_KIND, - ) - }) -}) - -function makeMemoryItem( - overrides: Partial = {}, -): FluidMemoryItem { - return { - id: "item_1", - workspaceId: "workspace_1", - kind: "stance", - payload: { statement: "s", scope: "scope", rationale: "r" }, - abstractL0: "abstract", - overviewL1: "overview", - sourceMessageId: null, - confidence: 0.8, - status: "active", - deactivationReason: null, - version: 1, - createdAt: new Date("2026-09-10T00:00:00Z"), - updatedAt: new Date("2026-09-10T00:00:00Z"), - ...overrides, - } -} diff --git a/src/domains/chat/memory-tools.ts b/src/domains/chat/memory-tools.ts deleted file mode 100644 index 63d48b11..00000000 --- a/src/domains/chat/memory-tools.ts +++ /dev/null @@ -1,66 +0,0 @@ -import type { - MemorySearchItem, - MemorySearchRequest, - MemorySearchResponse, - MemoryToolRuntime, -} from "@/agent-harness" -import { DISTILL_DEDUP_CANDIDATES_PER_KIND } from "@/domains/memory/distill-config" -import { tokenizeMemoryText } from "@/domains/memory/search-index" -import { memoryService } from "@/domains/memory/service" -import { fluidMemoryKinds, isFluidMemoryKind } from "@/domains/memory/types" -import type { FluidMemoryItem } from "@/infrastructure/db/schema" - -type NotebookMemoryToolsInput = { - readonly workspaceId: string -} - -export const notebookMemoryTools = { - createRuntime(input: NotebookMemoryToolsInput): MemoryToolRuntime { - return { - search: (request) => searchWorkspaceMemory(input.workspaceId, request), - } - }, -} as const - -async function searchWorkspaceMemory( - workspaceId: string, - request: MemorySearchRequest, -): Promise { - const tokens = tokenizeMemoryText(request.query).map((entry) => entry.token) - const kinds = request.kinds ?? fluidMemoryKinds - const items: MemorySearchItem[] = [] - - for (const kind of kinds) { - const candidates = await memoryService.findDedupCandidates( - workspaceId, - kind, - tokens, - // Same per-kind cap as the existing findDedupCandidates caller. - DISTILL_DEDUP_CANDIDATES_PER_KIND, - ) - for (const candidate of candidates) { - const item = toMemorySearchItem(candidate, items.length + 1) - if (item) items.push(item) - } - } - - return { - query: request.query, - items, - } -} - -function toMemorySearchItem( - item: FluidMemoryItem, - index: number, -): MemorySearchItem | null { - if (!isFluidMemoryKind(item.kind)) return null - - return { - ref: `mem:${index}`, - itemId: item.id, - kind: item.kind, - abstractL0: item.abstractL0, - overviewL1: item.overviewL1, - } -} diff --git a/src/domains/chat/prompt.ts b/src/domains/chat/prompt.ts index d61177ea..4f721019 100644 --- a/src/domains/chat/prompt.ts +++ b/src/domains/chat/prompt.ts @@ -16,8 +16,8 @@ import type { ChatHistoryMessage, SearchSources, } from "./contracts" +import { mementoMemoryTools } from "@/integrations/memento/memory-tools" import { notebookKnowhereTools } from "./knowhere-tools" -import { notebookMemoryTools } from "./memory-tools" const RECENT_CONTEXT_MESSAGE_LIMIT = 8 const CONTEXT_CONTENT_CHAR_LIMIT = 900 @@ -65,7 +65,7 @@ export const generateAgenticOutputManifestEffect = ( notebookKnowhereTools.createSearchOnlyRuntime({ searchSources: input.searchSources, }), - memoryTools: notebookMemoryTools.createRuntime({ + memoryTools: mementoMemoryTools.createRuntime({ workspaceId: input.workspaceId, }), ...(input.inspectImages ? { inspectImages: input.inspectImages } : {}), diff --git a/src/domains/chat/route-service.test.ts b/src/domains/chat/route-service.test.ts index 54e8dae0..d27b32dd 100644 --- a/src/domains/chat/route-service.test.ts +++ b/src/domains/chat/route-service.test.ts @@ -25,7 +25,8 @@ const mocks = vi.hoisted(() => ({ parsedStorageWriteAsset: vi.fn(), softDeleteChatThread: vi.fn(), startBackgroundReconciliation: vi.fn(), - triggerMemoryExtraction: vi.fn(), + captureMemoryTurn: vi.fn(), + recordActivations: vi.fn(), })) vi.mock("ai", async (importOriginal) => { @@ -63,8 +64,9 @@ vi.mock("@/domains/sources/background-reconcile", () => ({ startBackgroundReconciliation: mocks.startBackgroundReconciliation, })) -vi.mock("@/domains/memory/extract-trigger", () => ({ - triggerMemoryExtraction: mocks.triggerMemoryExtraction, +vi.mock("@/integrations/memento/client", () => ({ + captureMemoryTurn: mocks.captureMemoryTurn, + recordActivations: mocks.recordActivations, })) vi.mock("@/domains/sources/workflow-runtime", () => ({ @@ -853,11 +855,12 @@ describe("chat route services", () => { }) expect(result.status).toBe(200) - expect(mocks.triggerMemoryExtraction).toHaveBeenCalledWith({ + expect(mocks.captureMemoryTurn).toHaveBeenCalledWith({ workspaceId: workspace.id, - threadId: "thread_1", - userMessageId: "message_user", - assistantMessageId: "message_assistant", + sourceMessageId: "message_assistant", + userText: "Summarize it", + assistantText: "Summary", + referencedDocumentIds: [], }) expect(mocks.startBackgroundReconciliation).toHaveBeenCalledWith( workspace.id, diff --git a/src/domains/chat/service.test.ts b/src/domains/chat/service.test.ts index 0ac23870..133e0a54 100644 --- a/src/domains/chat/service.test.ts +++ b/src/domains/chat/service.test.ts @@ -83,23 +83,9 @@ describe("handleChatTurn", () => { }); }); - it("allows a turn with no local sources so remote retrieval can still run", async () => { - const retrieval = { - query: vi.fn().mockResolvedValue({ - results: [makeRetrievalResult()], - evidenceText: "Grounding content", - referencedChunks: [], - namespace: "notebook-namespace", - query: "What does the document say?", - routerUsed: "workflow_single_step", - answerText: null, - }), - }; + it("rejects a turn with no local sources without calling retrieval", async () => { + const retrieval = { query: vi.fn() }; const repository = makeRepository(); - const generateAnswer = vi.fn(async ({ searchSources }) => { - await searchSources({ query: "What does the document say?" }); - return makeHarnessRunResult("Grounded answer."); - }); const result = await handleChatTurn({ workspace: makeWorkspace(), @@ -107,13 +93,19 @@ describe("handleChatTurn", () => { question: "What does the document say?", excludedSourceIds: [], retrieval, - generateAnswer, + generateAnswer: vi.fn(), repository, }); - expect(Either.isRight(result)).toBe(true); - expect(generateAnswer).toHaveBeenCalled(); - expect(repository.appendMessageToThread).toHaveBeenCalled(); + expect(Either.isLeft(result)).toBe(true); + if (Either.isLeft(result)) { + expect(result.left).toMatchObject({ + status: 409, + message: "Upload and process a document before asking questions.", + }); + } + expect(retrieval.query).not.toHaveBeenCalled(); + expect(repository.appendMessageToThread).not.toHaveBeenCalled(); }); it("rejects chat before any source is ready without calling retrieval", async () => { diff --git a/src/domains/chat/service.ts b/src/domains/chat/service.ts index b2c5bce2..23bfa68d 100644 --- a/src/domains/chat/service.ts +++ b/src/domains/chat/service.ts @@ -85,7 +85,7 @@ export const handleChatTurnEffect = (input: ChatTurnInput) => const readySources = input.sources.filter( (source) => source.status === "ready" && source.knowhereDocumentId, ) - if (input.sources.length > 0 && readySources.length === 0) { + if (readySources.length === 0) { return yield* Effect.fail(noReadySources) } diff --git a/src/domains/memory/decay-candidates.test.ts b/src/domains/memory/decay-candidates.test.ts deleted file mode 100644 index 7635c1fb..00000000 --- a/src/domains/memory/decay-candidates.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { selectDecayCandidates } from "./decay-candidates" - -const NOW = new Date("2026-01-08T00:00:00Z") -const A_YEAR_AGO = new Date("2025-01-08T00:00:00Z") - -describe("selectDecayCandidates", () => { - it("flags an old, never-activated item below the threshold", () => { - const candidates = selectDecayCandidates({ - items: [{ id: "item_1", kind: "stance", createdAt: A_YEAR_AGO }], - activationsById: new Map(), - now: NOW, - scoreThreshold: 0.3, - }) - - expect(candidates).toEqual([ - { id: "item_1", kind: "stance", score: expect.any(Number), activationCount: 0 }, - ]) - expect(candidates[0]!.score).toBeLessThan(0.3) - }) - - it("does not flag a freshly created item even with no activations", () => { - const candidates = selectDecayCandidates({ - items: [{ id: "item_1", kind: "stance", createdAt: NOW }], - activationsById: new Map(), - now: NOW, - scoreThreshold: 0.3, - }) - - expect(candidates).toEqual([]) - }) - - it("does not flag an old item that was recently activated", () => { - const candidates = selectDecayCandidates({ - items: [{ id: "item_1", kind: "stance", createdAt: A_YEAR_AGO }], - activationsById: new Map([ - ["item_1", { activationCount: 3, lastActivatedAt: NOW }], - ]), - now: NOW, - scoreThreshold: 0.3, - }) - - expect(candidates).toEqual([]) - }) - - it("only flags items strictly below the given threshold", () => { - const items = [ - { id: "item_1", kind: "stance" as const, createdAt: A_YEAR_AGO }, - { id: "item_2", kind: "stance" as const, createdAt: NOW }, - ] - - const noneFlagged = selectDecayCandidates({ - items, - activationsById: new Map(), - now: NOW, - scoreThreshold: 0, - }) - expect(noneFlagged).toEqual([]) - - const allFlagged = selectDecayCandidates({ - items, - activationsById: new Map(), - now: NOW, - scoreThreshold: 1, - }) - expect(allFlagged.map((c) => c.id).sort()).toEqual(["item_1", "item_2"]) - }) -}) diff --git a/src/domains/memory/decay-candidates.ts b/src/domains/memory/decay-candidates.ts deleted file mode 100644 index 159176e6..00000000 --- a/src/domains/memory/decay-candidates.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { computeDecayScore } from "@/domains/retrieval-activation/decay-score" - -/** - * Confirmed decay-candidate threshold: at `BASE_HALF_LIFE_DAYS` (14), a - * never-activated item (ceiling 0.5) crosses this after ~24.3 days of - * silence; one activated once after ~60 days; one activated 3x after ~135 - * days. Chosen by simulating `computeDecayScore`'s actual day-counts across - * activation counts and confirming the resulting grace periods, not picked - * a priori — see the `记忆衰减聚类收尾方案` plan. - */ -export const DEFAULT_DECAY_SCORE_THRESHOLD = 0.15 - -export type DecayableItem = { - readonly id: string - /** Raw `fluid_memory_items.kind` column value — carried through, not validated. */ - readonly kind: string - readonly createdAt: Date -} - -export type ItemActivationStats = { - readonly activationCount: number - readonly lastActivatedAt: Date | null -} - -export type DecayCandidate = { - readonly id: string - readonly kind: string - readonly score: number - readonly activationCount: number -} - -/** - * Pure selection: given active fluid_memory items and their (possibly - * absent) activation ledger rows, return the ones whose decay score is - * below `scoreThreshold`. Does not decide the threshold itself and does not - * write anything — per the plan, a decay score crossing the line only - * produces a *candidate*; moving it to `inactive` is a separate, explicit - * step (see `memoryRepository.deactivateDecayedItemsEffect`). - * - * The anchor for an item with no ledger row (never activated) is its own - * `createdAt` — a real, meaningful signal here (unlike a crystal chunk, - * where "no row" means "no signal at all"), so a never-activated item still - * decays normally from the moment it was created. - */ -export function selectDecayCandidates(input: { - readonly items: readonly DecayableItem[] - readonly activationsById: ReadonlyMap - readonly now: Date - readonly scoreThreshold: number -}): readonly DecayCandidate[] { - return input.items.flatMap((item) => { - const activation = input.activationsById.get(item.id) - const activationCount = activation?.activationCount ?? 0 - const anchorAt = activation?.lastActivatedAt ?? item.createdAt - const score = computeDecayScore({ - activationCount, - anchorAt, - now: input.now, - }) - return score < input.scoreThreshold - ? [{ id: item.id, kind: item.kind, score, activationCount }] - : [] - }) -} diff --git a/src/domains/memory/distill-config.ts b/src/domains/memory/distill-config.ts deleted file mode 100644 index 4c82abbf..00000000 --- a/src/domains/memory/distill-config.ts +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Distill job defaults from the two-tier fluid-memory plan. - * Tunable later; kept as named constants (not scattered literals). - */ - -/** Process-local cooldown + QStash workflowRunId bucket width (mirrors reconcile). */ -export const DISTILL_COOLDOWN_MS = 5 * 60_000 - -/** Capture may trigger distill once pending observations reach this count. */ -export const DISTILL_MIN_PENDING = 8 - -/** Max pending rows claimed per distill run (oldest first). */ -export const DISTILL_BATCH_MAX = 40 - -/** Lexical dedup candidates loaded per memory kind for one distill batch. */ -export const DISTILL_DEDUP_CANDIDATES_PER_KIND = 8 - -/** Delete consumed observations older than this (retention sweep after distill). */ -export const DISTILL_CONSUMED_RETENTION_MS = 30 * 24 * 60_000 diff --git a/src/domains/memory/distill-model.ts b/src/domains/memory/distill-model.ts deleted file mode 100644 index ef2485ff..00000000 --- a/src/domains/memory/distill-model.ts +++ /dev/null @@ -1,83 +0,0 @@ -import "server-only" - -import { generateObject } from "ai" - -import { buildDistillPrompt } from "./distill-prompts" -import { - entityDistillOutputSchema, - experienceDistillOutputSchema, - indicatorDistillOutputSchema, - toMemoryOperations, - type DistillObservationInput, - type DistillPassKind, - type ExistingMemoryContextItem, - type MemoryOperations, -} from "./distill-types" -import { CHAT_MODEL } from "@/lib/ai" -import { summarizeUnknownError } from "@/lib/format-log-value" -import { logger } from "@/lib/logger" - -const MEMORY_EXTRACTION_MODEL = process.env.MEMORY_EXTRACTION_MODEL ?? CHAT_MODEL - -/** - * One structured-output call for a single distill pass. - * Best-effort — distill runs as a background job; a model failure returns - * null so the workflow can skip applying that pass (logged). No multi-level - * fallback chain. - * - * Input: pending observation batch + existing memories of kinds this pass may - * write + allowed document ids for the batch. - * Output: full MemoryOperations with only this pass's arrays populated - * (others empty), ready for resolveMemoryOperations. - */ -export async function distillMemoryPass(input: { - readonly pass: DistillPassKind - readonly workspaceId: string - readonly observations: readonly DistillObservationInput[] - readonly existingItems: readonly ExistingMemoryContextItem[] - readonly referencedDocumentIds: readonly string[] -}): Promise { - const prompt = buildDistillPrompt(input.pass, { - observations: input.observations, - existingItems: input.existingItems, - referencedDocumentIds: input.referencedDocumentIds, - }) - - try { - switch (input.pass) { - case "indicator": { - const response = await generateObject({ - model: MEMORY_EXTRACTION_MODEL, - schema: indicatorDistillOutputSchema, - messages: [{ role: "user", content: prompt }], - }) - return toMemoryOperations("indicator", response.object) - } - case "experience": { - const response = await generateObject({ - model: MEMORY_EXTRACTION_MODEL, - schema: experienceDistillOutputSchema, - messages: [{ role: "user", content: prompt }], - }) - return toMemoryOperations("experience", response.object) - } - case "entity": { - const response = await generateObject({ - model: MEMORY_EXTRACTION_MODEL, - schema: entityDistillOutputSchema, - messages: [{ role: "user", content: prompt }], - }) - return toMemoryOperations("entity", response.object) - } - } - } catch (error) { - logger.warn("memory: distill model call failed; skipping pass", { - workspaceId: input.workspaceId, - pass: input.pass, - model: MEMORY_EXTRACTION_MODEL, - observationCount: input.observations.length, - error: summarizeUnknownError(error), - }) - return null - } -} diff --git a/src/domains/memory/distill-prompts.test.ts b/src/domains/memory/distill-prompts.test.ts deleted file mode 100644 index 94578800..00000000 --- a/src/domains/memory/distill-prompts.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { buildDistillPrompt } from "./distill-prompts" -import type { DistillObservationInput } from "./distill-types" - -const observations: DistillObservationInput[] = [ - { - id: "obs-1", - signal: "看重毛利率", - evidenceQuote: "毛利率是核心", - subjectHint: "毛利率", - confidence: 0.9, - referencedDocumentIds: ["doc-1"], - }, - { - id: "obs-2", - signal: "跟踪英伟达", - evidenceQuote: "英伟达一直在跟踪", - subjectHint: "英伟达", - confidence: 0.8, - referencedDocumentIds: [], - }, -] - -describe("buildDistillPrompt", () => { - it("indicator pass: only indicator schema, no other kind arrays", () => { - const prompt = buildDistillPrompt("indicator", { - observations, - existingItems: [ - { - id: "item-1", - kind: "indicator_pref", - abstractL0: "看重毛利率", - payloadSummary: "毛利率 — 毛利占营收", - }, - ], - referencedDocumentIds: ["doc-1"], - }) - - expect(prompt).toContain("You DISTILL indicator preferences") - expect(prompt).toContain('"indicatorPrefs"') - expect(prompt).not.toContain('"stances"') - expect(prompt).not.toContain('"decisionRules"') - expect(prompt).not.toContain('"entities"') - expect(prompt).toContain("never emit") - expect(prompt).toContain("PENDING OBSERVATIONS") - expect(prompt).toContain("id=obs-1") - expect(prompt).toContain("id=obs-2") - expect(prompt).toContain("看重毛利率") - expect(prompt).toContain("跟踪英伟达") - expect(prompt).toContain("id=item-1") - expect(prompt).toContain("doc-1") - // All observations go to every pass — no kind routing. - expect(prompt.indexOf("obs-1")).toBeLessThan(prompt.indexOf("obs-2")) - }) - - it("experience pass: stances+rules only; still sees full observation batch", () => { - const prompt = buildDistillPrompt("experience", { - observations, - existingItems: [], - referencedDocumentIds: [], - }) - - expect(prompt).toContain("You DISTILL stances and decision rules") - expect(prompt).toContain('"stances"') - expect(prompt).toContain('"decisionRules"') - expect(prompt).not.toContain('"indicatorPrefs"') - expect(prompt).not.toContain('"entities"') - expect(prompt).toContain("id=obs-1") - expect(prompt).toContain("id=obs-2") - expect(prompt).toContain("(no existing memories yet)") - expect(prompt).toContain("(no documents referenced in this batch)") - }) - - it("entity pass: entities only; referenced ids from batch", () => { - const prompt = buildDistillPrompt("entity", { - observations, - existingItems: [ - { - id: "item-4", - kind: "entity_of_interest", - abstractL0: "跟踪英伟达", - payloadSummary: "英伟达 NVDA", - }, - ], - referencedDocumentIds: ["doc-1"], - }) - - expect(prompt).toContain("You DISTILL entities of interest") - expect(prompt).toContain('"entities"') - expect(prompt).not.toContain('"indicatorPrefs"') - expect(prompt).not.toContain('"stances"') - expect(prompt).not.toContain('"decisionRules"') - expect(prompt).toContain("never invent ids") - expect(prompt).toContain("id=item-4") - expect(prompt).toContain("doc-1") - }) - - it("keeps illustrative examples separated from main instructions", () => { - const prompt = buildDistillPrompt("indicator", { - observations: [], - existingItems: [], - referencedDocumentIds: [], - }) - const main = prompt.slice(0, prompt.indexOf("## Illustrative examples")) - expect(main).not.toMatch(/gross margin|市盈率|\bPE\b|Fed|美联储|NVIDIA|英伟达/i) - expect(prompt).toContain("## Illustrative examples (finance vertical") - expect(prompt).toContain("(no pending observations)") - }) -}) diff --git a/src/domains/memory/distill-prompts.ts b/src/domains/memory/distill-prompts.ts deleted file mode 100644 index 3e9e70ca..00000000 --- a/src/domains/memory/distill-prompts.ts +++ /dev/null @@ -1,311 +0,0 @@ -import type { - DistillObservationInput, - DistillPassKind, - ExistingMemoryContextItem, -} from "./distill-types" - -/** - * Distill prompts — three isolated passes over the same pending observation - * batch. Each pass sees ALL observations (no kind routing) and only the - * existing memories of kinds that pass may write. - */ - -const INDICATOR_OUTPUT_SCHEMA_BLOCK = `{ - "indicatorPrefs": [{ - "name": "string", - "aliases": ["string"], - "definition": "string", - "polarity": "higher_better|lower_better|context", - "importance": "core|secondary", - "formulaHint": "string (optional — omit if none)", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }] -}` - -const EXPERIENCE_OUTPUT_SCHEMA_BLOCK = `{ - "stances": [{ - "statement": "string (the stance text; do not use a name field)", - "scope": "string", - "rationale": "string", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }], - "decisionRules": [{ - "when": "string", - "then": "string", - "priority": "high|medium|low", - "rationale": "string", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }] -}` - -const ENTITY_OUTPUT_SCHEMA_BLOCK = `{ - "entities": [{ - "name": "string", - "ticker": "string optional", - "aliases": ["string"], - "knowhereDocumentIds": ["only ids listed under REFERENCED DOCUMENT IDS"], - "reason": "string", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }] -}` - -const INDICATOR_ILLUSTRATIVE_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) - -These show shape and judgement only. Distill whatever the observations support; -do not force the batch into this domain or these metric names. - -- Recurring evaluation metric named across clues → one indicatorPref (stable name + definition + polarity + importance). -- Same metric restated with a nuance → merge into the existing item, do not create a second. -- Skip: a one-off number question, document fact, or weak single-mention with no reusable criterion.` - -const EXPERIENCE_ILLUSTRATIVE_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) - -These show shape and judgement only. Distill whatever the observations support; -do not force the batch into this domain. - -- Durable judgement frame (e.g. long-horizon) → one stance (statement + scope + rationale). -- Reusable when → then discipline over the user's criteria → one decisionRule. -- Abstract away one-off instances; keep a single intent per rule. Split unrelated intents. -- Skip: process narration, document facts, or a preference that is only a metric definition (indicators are another pass).` - -const ENTITY_ILLUSTRATIVE_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) - -These show shape and judgement only. Distill whatever the observations support; -do not force the batch into this domain. - -- User actively tracks a named company/issuer across clues → one entity (name + reason; optional ticker/aliases). -- Same subject restated → merge; attach knowhereDocumentIds only from REFERENCED DOCUMENT IDS. -- Skip: a company mentioned only as a one-off fact question, or names that are not subjects of ongoing interest.` - -const INDICATOR_INSTRUCTIONS_BLOCK = `You DISTILL indicator preferences for a user's fluid memory. - -You receive a BATCH of raw observations (cheap clues about what the USER cares -about) plus existing indicator memories. Produce durable indicator_pref items -only. A separate pass handles stances, decision rules, and entities — never emit -those kinds here. - -Constraints: -- One stable topic/name per preference; merge overlapping or synonymous names. -- Capture "what the user repeatedly uses to evaluate", not one-off facts. -- Keep unrelated criteria as separate items; do not mix them into one payload. - -## What to emit - -indicatorPrefs — recurring metrics or criteria the user uses to evaluate things. -Fields: name, aliases, definition, polarity (higher_better | lower_better | context), -importance (core | secondary), optional formulaHint, abstractL0, overviewL1, -confidence, decision. - -## Language - -- Keep this instruction set and enum/field names in English. -- Write every free-text value in the same language the observations use. Do not - translate the user's terms into English unless the observations themselves used English. - -## Decision rules - -- Distill only from the observation batch evidence about the USER. -- Skip document facts, retrieved numbers, and weak/ephemeral clues. -- For every candidate, choose exactly one op against EXISTING MEMORIES: - - create — genuinely new indicator - - skip — already covered, or too weak - - merge — same indicator refined; emit the full merged fields and set targetItemId - - deprecate — user clearly reversed a stored indicator; set targetItemId -- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. -- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when explicitly stated. -- Omit optional fields instead of setting them to null. -- If nothing qualifies, return {"indicatorPrefs": []}.` - -const EXPERIENCE_INSTRUCTIONS_BLOCK = `You DISTILL stances and decision rules (insights) for a user's fluid memory. - -You receive a BATCH of raw observations plus existing stance/decision-rule -memories. Produce durable stances and decisionRules only. A separate pass -handles indicators and entities — never emit those kinds here. - -Constraints: -- Generalizable, reusable insight — not a process log of one session. -- Atomic scope: one intent per decisionRule; split if when would mix goals. -- Abstract away specific one-off entities/ids from the situation framing when the - rule itself is general; keep concrete names only when the insight requires them. -- Do not restate a bare metric definition as a decisionRule — that belongs to the indicator pass. - -## What to emit - -- stances — durable positions that shape how the user weighs evidence. - Fields: statement (required; do not invent a "name" field), scope, rationale, - abstractL0, overviewL1, confidence, decision. -- decisionRules — reusable when → then disciplines the user stated or clearly endorsed. - Fields: when, then, priority (high | medium | low), rationale, abstractL0, - overviewL1, confidence, decision. - -## Language - -- Keep this instruction set and enum/field names in English. -- Write every free-text value in the same language the observations use. Do not - translate the user's terms into English unless the observations themselves used English. - -## Decision rules - -- Distill only from the observation batch evidence about the USER. -- Skip document facts, small talk, and weak/ephemeral clues. -- For every candidate, choose exactly one op against EXISTING MEMORIES: - - create — genuinely new - - skip — already covered, or too weak - - merge — same insight refined; emit the full merged fields and set targetItemId - - deprecate — user clearly reversed a stored item; set targetItemId -- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. -- Prefer one record per insight. Do not invent a near-duplicate decisionRule for a stance that already encodes the same frame unless the user stated an explicit when → then action. -- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when explicitly stated. -- Omit optional fields instead of setting them to null. -- If nothing qualifies, return {"stances": [], "decisionRules": []}.` - -const ENTITY_INSTRUCTIONS_BLOCK = `You DISTILL entities of interest for a user's fluid memory. - -You receive a BATCH of raw observations plus existing entity memories. Produce -durable entity_of_interest items only. A separate pass handles indicators, -stances, and decision rules — never emit those kinds here. - -Constraints: -- Stable card for a subject the USER actively tracks. -- Merge overlapping names/aliases into one item; keep unrelated subjects separate. -- Attach document provenance only from ids listed under REFERENCED DOCUMENT IDS. - -## What to emit - -entities — named subjects the user is actively tracking. -Fields: name, optional ticker, aliases, reason (required), knowhereDocumentIds -(only from REFERENCED DOCUMENT IDS below; never invent ids), abstractL0, -overviewL1, confidence, decision. - -## Language - -- Keep this instruction set and enum/field names in English. -- Write every free-text value in the same language the observations use. Do not - translate the user's terms into English unless the observations themselves used English. - -## Decision rules - -- Distill only from the observation batch evidence about the USER. -- Skip one-off name drops, document facts, and weak/ephemeral mentions. -- For every candidate, choose exactly one op against EXISTING MEMORIES: - - create — genuinely new tracked subject - - skip — already covered, or too weak - - merge — same subject refined; emit the full merged fields and set targetItemId - - deprecate — user clearly stopped tracking / reversed; set targetItemId -- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. -- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when explicitly stated. -- Omit optional fields instead of setting them to null. -- If nothing qualifies, return {"entities": []}.` - -export type BuildDistillPromptInput = { - readonly observations: readonly DistillObservationInput[] - readonly existingItems: readonly ExistingMemoryContextItem[] - readonly referencedDocumentIds: readonly string[] -} - -export function buildDistillPrompt( - pass: DistillPassKind, - input: BuildDistillPromptInput, -): string { - switch (pass) { - case "indicator": - return assemblePrompt({ - instructions: INDICATOR_INSTRUCTIONS_BLOCK, - examples: INDICATOR_ILLUSTRATIVE_BLOCK, - outputSchema: INDICATOR_OUTPUT_SCHEMA_BLOCK, - input, - }) - case "experience": - return assemblePrompt({ - instructions: EXPERIENCE_INSTRUCTIONS_BLOCK, - examples: EXPERIENCE_ILLUSTRATIVE_BLOCK, - outputSchema: EXPERIENCE_OUTPUT_SCHEMA_BLOCK, - input, - }) - case "entity": - return assemblePrompt({ - instructions: ENTITY_INSTRUCTIONS_BLOCK, - examples: ENTITY_ILLUSTRATIVE_BLOCK, - outputSchema: ENTITY_OUTPUT_SCHEMA_BLOCK, - input, - }) - } -} - -function assemblePrompt(args: { - readonly instructions: string - readonly examples: string - readonly outputSchema: string - readonly input: BuildDistillPromptInput -}): string { - const existingBlock = - args.input.existingItems.length === 0 - ? "(no existing memories yet)" - : args.input.existingItems - .map( - (item) => - `- [${item.kind}] id=${item.id} :: ${item.abstractL0} :: ${item.payloadSummary}`, - ) - .join("\n") - - const documentsBlock = - args.input.referencedDocumentIds.length === 0 - ? "(no documents referenced in this batch)" - : args.input.referencedDocumentIds.join(", ") - - const observationsBlock = - args.input.observations.length === 0 - ? "(no pending observations)" - : args.input.observations - .map((observation) => formatObservation(observation)) - .join("\n\n") - - return `${args.instructions} - -${args.examples} - -## Output JSON schema (follow exactly; do not invent fields) - -${args.outputSchema} - -## EXISTING MEMORIES - -${existingBlock} - -## REFERENCED DOCUMENT IDS - -${documentsBlock} - -## PENDING OBSERVATIONS - -${observationsBlock}` -} - -function formatObservation(observation: DistillObservationInput): string { - const subject = - observation.subjectHint && observation.subjectHint.length > 0 - ? observation.subjectHint - : "(none)" - const docs = - observation.referencedDocumentIds.length === 0 - ? "(none)" - : observation.referencedDocumentIds.join(", ") - return `- id=${observation.id} - signal: ${observation.signal} - evidenceQuote: ${observation.evidenceQuote} - subjectHint: ${subject} - confidence: ${observation.confidence} - referencedDocumentIds: ${docs}` -} diff --git a/src/domains/memory/distill-trigger.test.ts b/src/domains/memory/distill-trigger.test.ts deleted file mode 100644 index f4a09d10..00000000 --- a/src/domains/memory/distill-trigger.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest" - -import { DISTILL_COOLDOWN_MS, DISTILL_MIN_PENDING } from "./distill-config" - -const mocks = vi.hoisted(() => ({ - loggerError: vi.fn(), - loggerInfo: vi.fn(), - loggerWarn: vi.fn(), - trigger: vi.fn(), - countPendingObservations: vi.fn(), -})) - -vi.mock("@upstash/workflow", () => ({ - Client: class { - trigger = mocks.trigger - }, -})) - -vi.mock("@/lib/logger", () => ({ - logger: { - error: mocks.loggerError, - info: mocks.loggerInfo, - warn: mocks.loggerWarn, - }, -})) - -vi.mock("./service", () => ({ - memoryService: { - countPendingObservations: mocks.countPendingObservations, - }, -})) - -describe("triggerMemoryDistill", () => { - afterEach(async () => { - vi.clearAllMocks() - vi.useRealTimers() - delete process.env.QSTASH_TOKEN - delete process.env.NOTEBOOK_PUBLIC_URL - const { resetMemoryDistillTriggerStateForTests } = await import( - "./distill-trigger" - ) - resetMemoryDistillTriggerStateForTests() - vi.resetModules() - }) - - it("does not trigger when pending is below the plan threshold", async () => { - mocks.countPendingObservations.mockResolvedValue(DISTILL_MIN_PENDING - 1) - process.env.QSTASH_TOKEN = "qstash_token" - - const { triggerMemoryDistill } = await import("./distill-trigger") - await triggerMemoryDistill({ workspaceId: "workspace_1" }) - - expect(mocks.trigger).not.toHaveBeenCalled() - }) - - it("deduplicates workflow triggers only within a bounded cooldown", async () => { - vi.useFakeTimers() - vi.setSystemTime(new Date("2026-06-30T00:00:00.000Z")) - process.env.QSTASH_TOKEN = "qstash_token" - process.env.NOTEBOOK_PUBLIC_URL = "https://notebook.example" - mocks.countPendingObservations.mockResolvedValue(DISTILL_MIN_PENDING) - mocks.trigger.mockResolvedValue({}) - - const { triggerMemoryDistill } = await import("./distill-trigger") - - await triggerMemoryDistill({ workspaceId: "workspace_1" }) - await triggerMemoryDistill({ workspaceId: "workspace_1" }) - - expect(mocks.trigger).toHaveBeenCalledTimes(1) - expect(mocks.trigger).toHaveBeenLastCalledWith({ - url: "https://notebook.example/api/memory/distill", - body: { workspaceId: "workspace_1" }, - workflowRunId: `workspace_1-${Math.floor( - new Date("2026-06-30T00:00:00.000Z").getTime() / DISTILL_COOLDOWN_MS, - )}`, - retries: 3, - }) - - vi.setSystemTime(new Date("2026-06-30T00:05:01.000Z")) - await triggerMemoryDistill({ workspaceId: "workspace_1" }) - - expect(mocks.trigger).toHaveBeenCalledTimes(2) - }) -}) diff --git a/src/domains/memory/distill-trigger.ts b/src/domains/memory/distill-trigger.ts deleted file mode 100644 index a9bd0fd6..00000000 --- a/src/domains/memory/distill-trigger.ts +++ /dev/null @@ -1,77 +0,0 @@ -import "server-only" - -import { Client } from "@upstash/workflow" - -import { DISTILL_COOLDOWN_MS, DISTILL_MIN_PENDING } from "./distill-config" -import type { MemoryDistillPayload } from "./distill-workflow" -import { memoryService } from "./service" -import { logger } from "@/lib/logger" - -// Re-trigger protection: process-local cooldown + bucketed workflowRunId. -// Mirrors background-reconcile — same cooldown width keys both guards. - -const lastTriggeredAtByWorkspaceId: Map = new Map() - -function resolveBaseURL(): string { - return process.env.NOTEBOOK_PUBLIC_URL ?? "http://localhost:3000" -} - -/** - * Fire-and-forget distill trigger for one workspace. - * Caller should already know pending may be high; this re-checks the count, - * applies cooldown + bucketed workflowRunId, then enqueues QStash. - */ -export async function triggerMemoryDistill( - payload: MemoryDistillPayload, -): Promise { - const pendingCount = await memoryService.countPendingObservations( - payload.workspaceId, - ) - if (pendingCount < DISTILL_MIN_PENDING) return - - const now = Date.now() - const lastTriggeredAt = lastTriggeredAtByWorkspaceId.get(payload.workspaceId) - if ( - lastTriggeredAt !== undefined && - now - lastTriggeredAt < DISTILL_COOLDOWN_MS - ) { - return - } - lastTriggeredAtByWorkspaceId.set(payload.workspaceId, now) - - const token = process.env.QSTASH_TOKEN - if (!token) { - logger.warn("memory: skipping distill — QSTASH_TOKEN not set", { - workspaceId: payload.workspaceId, - pendingCount, - }) - lastTriggeredAtByWorkspaceId.delete(payload.workspaceId) - return - } - - const url = `${resolveBaseURL()}/api/memory/distill` - try { - await new Client({ token }).trigger({ - url, - body: payload, - workflowRunId: `${payload.workspaceId}-${Math.floor(now / DISTILL_COOLDOWN_MS)}`, - retries: 3, - }) - logger.info("memory: distill workflow triggered", { - workspaceId: payload.workspaceId, - pendingCount, - url, - }) - } catch (error) { - lastTriggeredAtByWorkspaceId.delete(payload.workspaceId) - logger.error("memory: failed to trigger distill workflow", { - workspaceId: payload.workspaceId, - message: error instanceof Error ? error.message : String(error), - }) - } -} - -/** Test helper: clear process-local cooldown map between cases. */ -export function resetMemoryDistillTriggerStateForTests(): void { - lastTriggeredAtByWorkspaceId.clear() -} diff --git a/src/domains/memory/distill-types.test.ts b/src/domains/memory/distill-types.test.ts deleted file mode 100644 index b3b161d5..00000000 --- a/src/domains/memory/distill-types.test.ts +++ /dev/null @@ -1,209 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { - entityDistillOutputSchema, - experienceDistillOutputSchema, - indicatorDistillOutputSchema, - memoryOperationsSchema, - summarizePayloadForContext, - toMemoryOperations, -} from "./distill-types" - -describe("distill output schemas", () => { - it("indicator pass accepts prefs and defaults empty array", () => { - expect(indicatorDistillOutputSchema.parse({})).toEqual({ - indicatorPrefs: [], - }) - const parsed = indicatorDistillOutputSchema.parse({ - indicatorPrefs: [ - { - name: "毛利率", - aliases: [], - definition: "毛利占营收", - polarity: "higher_better", - importance: "core", - abstractL0: "看重毛利率", - overviewL1: "用户用毛利率判断质量。", - confidence: 0.9, - decision: { op: "create" }, - }, - ], - }) - expect(parsed.indicatorPrefs).toHaveLength(1) - expect(parsed).not.toHaveProperty("stances") - }) - - it("experience pass accepts stance+rule when required fields are present", () => { - const parsed = experienceDistillOutputSchema.parse({ - stances: [ - { - statement: "长期持有", - scope: "投资 horizon", - rationale: "用户明确说长期", - abstractL0: "长期持有", - overviewL1: "用户以长期视角评估。", - confidence: 1, - decision: { op: "create" }, - }, - ], - decisionRules: [ - { - when: "毛利率连续两季下滑", - then: "减仓观望", - priority: "high", - rationale: "用户自述纪律", - abstractL0: "毛利率下滑则减仓", - overviewL1: "连续两季下滑时减仓观望。", - confidence: 0.95, - decision: { op: "create" }, - }, - ], - }) - expect(parsed.stances[0]?.statement).toBe("长期持有") - expect(parsed.decisionRules).toHaveLength(1) - expect(parsed).not.toHaveProperty("indicatorPrefs") - }) - - it("rejects stance that uses name instead of statement (no coerce)", () => { - expect(() => - experienceDistillOutputSchema.parse({ - stances: [ - { - name: "长期持有", - scope: "投资", - rationale: "用户明确说长期", - abstractL0: "长期持有", - overviewL1: "用户以长期视角评估。", - confidence: 1, - decision: { op: "create" }, - }, - ], - }), - ).toThrow() - }) - - it("rejects entity missing reason (no fill from abstractL0)", () => { - expect(() => - entityDistillOutputSchema.parse({ - entities: [ - { - name: "英伟达", - aliases: ["NVIDIA"], - knowhereDocumentIds: ["doc-1"], - abstractL0: "持续跟踪英伟达", - overviewL1: "用户把英伟达列为跟踪标的。", - confidence: 0.8, - decision: { op: "merge", targetItemId: "item-4" }, - }, - ], - }), - ).toThrow() - }) - - it("entity pass accepts when reason is present", () => { - const parsed = entityDistillOutputSchema.parse({ - entities: [ - { - name: "英伟达", - aliases: ["NVIDIA"], - knowhereDocumentIds: ["doc-1"], - reason: "用户持续跟踪", - abstractL0: "持续跟踪英伟达", - overviewL1: "用户把英伟达列为跟踪标的。", - confidence: 0.8, - decision: { op: "merge", targetItemId: "item-4" }, - }, - ], - }) - expect(parsed.entities[0]?.reason).toBe("用户持续跟踪") - }) - - it("toMemoryOperations expands each pass into the full four-array record", () => { - expect( - toMemoryOperations("indicator", { - indicatorPrefs: [], - }), - ).toEqual({ - indicatorPrefs: [], - stances: [], - decisionRules: [], - entities: [], - }) - - const experience = toMemoryOperations( - "experience", - experienceDistillOutputSchema.parse({ - stances: [ - { - statement: "长期", - scope: "投资", - rationale: "用户说的", - abstractL0: "长期", - overviewL1: "长期视角。", - confidence: 1, - decision: { op: "create" }, - }, - ], - }), - ) - expect(experience.stances).toHaveLength(1) - expect(experience.indicatorPrefs).toEqual([]) - expect(experience.entities).toEqual([]) - - const entity = toMemoryOperations( - "entity", - entityDistillOutputSchema.parse({ - entities: [ - { - name: "英伟达", - aliases: [], - knowhereDocumentIds: [], - reason: "跟踪", - abstractL0: "跟踪英伟达", - overviewL1: "用户跟踪英伟达。", - confidence: 0.7, - decision: { op: "create" }, - }, - ], - }), - ) - expect(entity.entities).toHaveLength(1) - expect(entity.decisionRules).toEqual([]) - }) - - it("memoryOperationsSchema parses the full four-array shape", () => { - const parsed = memoryOperationsSchema.parse({}) - expect(parsed).toEqual({ - indicatorPrefs: [], - stances: [], - decisionRules: [], - entities: [], - }) - }) -}) - -describe("summarizePayloadForContext", () => { - it("summarizes each kind for existing-memory context lines", () => { - expect( - summarizePayloadForContext("indicator_pref", { - name: "毛利率", - definition: "毛利占营收", - }), - ).toBe("毛利率 — 毛利占营收") - expect( - summarizePayloadForContext("stance", { statement: "长期持有" }), - ).toBe("长期持有") - expect( - summarizePayloadForContext("decision_rule", { - when: "下滑", - then: "减仓", - }), - ).toBe("下滑 => 减仓") - expect( - summarizePayloadForContext("entity_of_interest", { - name: "英伟达", - ticker: "NVDA", - }), - ).toBe("英伟达 NVDA") - }) -}) diff --git a/src/domains/memory/distill-types.ts b/src/domains/memory/distill-types.ts deleted file mode 100644 index 44b22749..00000000 --- a/src/domains/memory/distill-types.ts +++ /dev/null @@ -1,190 +0,0 @@ -import { z } from "zod" - -import { - decisionRulePayloadSchema, - entityOfInterestPayloadSchema, - indicatorPreferencePayloadSchema, - stancePayloadSchema, - type FluidMemoryKind, -} from "./types" - -/** - * Distill-pass LLM contracts. - * - * Three separate structured-output schemas (indicator / experience / - * entity). Capture never emits these shapes — distill is the only writer - * of create/merge/deprecate decisions over `fluid_memory_items`. - */ - -function nullToUndefined(value: unknown): unknown { - return value === null ? undefined : value -} - -const decisionSchema = z.object({ - op: z.enum(["create", "skip", "merge", "deprecate"]), - targetItemId: z.preprocess( - nullToUndefined, - z - .string() - .optional() - .describe( - "Required for merge/deprecate: id of the existing memory item. Omit for create/skip.", - ), - ), - reason: z.preprocess( - nullToUndefined, - z - .string() - .optional() - .describe("Short justification, especially for skip/merge/deprecate."), - ), -}) - -const memorySidecarFields = { - abstractL0: z - .string() - .min(1) - .describe("One line, <= 30 words: the essence of this insight."), - overviewL1: z - .string() - .min(1) - .describe("2-3 sentences: what it means and when it applies."), - confidence: z - .number() - .min(0) - .max(1) - .describe("How explicitly the user stated this across the batch (1 = explicit)."), - decision: decisionSchema, -} - -const stanceEntrySchema = stancePayloadSchema.extend(memorySidecarFields) - -const entityEntrySchema = - entityOfInterestPayloadSchema.extend(memorySidecarFields) - -const indicatorEntrySchema = - indicatorPreferencePayloadSchema.extend(memorySidecarFields) - -const decisionRuleEntrySchema = - decisionRulePayloadSchema.extend(memorySidecarFields) - -/** Full four-array shape consumed by resolveMemoryOperations. */ -export const memoryOperationsSchema = z.object({ - indicatorPrefs: z.array(indicatorEntrySchema).default([]), - stances: z.array(stanceEntrySchema).default([]), - decisionRules: z.array(decisionRuleEntrySchema).default([]), - entities: z.array(entityEntrySchema).default([]), -}) - -export type MemoryOperations = z.infer - -/** Pass 1 — indicator preferences only. */ -export const indicatorDistillOutputSchema = z.object({ - indicatorPrefs: z.array(indicatorEntrySchema).default([]), -}) - -/** Pass 2 — stances + decision rules. */ -export const experienceDistillOutputSchema = z.object({ - stances: z.array(stanceEntrySchema).default([]), - decisionRules: z.array(decisionRuleEntrySchema).default([]), -}) - -/** Pass 3 — entities of interest only. */ -export const entityDistillOutputSchema = z.object({ - entities: z.array(entityEntrySchema).default([]), -}) - -export type IndicatorDistillOutput = z.infer -export type ExperienceDistillOutput = z.infer< - typeof experienceDistillOutputSchema -> -export type EntityDistillOutput = z.infer - -export const distillPassKinds = [ - "indicator", - "experience", - "entity", -] as const - -export type DistillPassKind = (typeof distillPassKinds)[number] - -/** Pending observation row shape fed into distill prompts (batch evidence). */ -export type DistillObservationInput = { - readonly id: string - readonly signal: string - readonly evidenceQuote: string - readonly subjectHint: string | null - readonly confidence: number - readonly referencedDocumentIds: readonly string[] -} - -export type ExistingMemoryContextItem = { - readonly id: string - readonly kind: FluidMemoryKind - readonly abstractL0: string - readonly payloadSummary: string -} - -/** Expand a single-pass LLM object into the full MemoryOperations record. */ -export function toMemoryOperations( - pass: DistillPassKind, - output: - | IndicatorDistillOutput - | ExperienceDistillOutput - | EntityDistillOutput, -): MemoryOperations { - switch (pass) { - case "indicator": { - const typed = output as IndicatorDistillOutput - return { - indicatorPrefs: typed.indicatorPrefs, - stances: [], - decisionRules: [], - entities: [], - } - } - case "experience": { - const typed = output as ExperienceDistillOutput - return { - indicatorPrefs: [], - stances: typed.stances, - decisionRules: typed.decisionRules, - entities: [], - } - } - case "entity": { - const typed = output as EntityDistillOutput - return { - indicatorPrefs: [], - stances: [], - decisionRules: [], - entities: typed.entities, - } - } - } -} - -/** Compact payload label for existing-item context in distill prompts. */ -export function summarizePayloadForContext( - kind: FluidMemoryKind, - payload: unknown, -): string { - if (!payload || typeof payload !== "object") return "" - const record = payload as Record - switch (kind) { - case "indicator_pref": - return [record.name, record.definition] - .filter((part) => typeof part === "string" && part.length > 0) - .join(" — ") - case "stance": - return typeof record.statement === "string" ? record.statement : "" - case "decision_rule": - return [record.when, record.then] - .filter((part) => typeof part === "string" && part.length > 0) - .join(" => ") - case "entity_of_interest": - return [record.name, record.ticker] - .filter((part) => typeof part === "string" && part.length > 0) - .join(" ") - } -} diff --git a/src/domains/memory/distill-workflow.test.ts b/src/domains/memory/distill-workflow.test.ts deleted file mode 100644 index a96b576a..00000000 --- a/src/domains/memory/distill-workflow.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { - normalizeMemoryDistillPayload, - toDistillObservationInput, -} from "./distill-workflow" -import type { FluidObservation } from "@/infrastructure/db/schema" - -describe("normalizeMemoryDistillPayload", () => { - it("accepts a workspace id", () => { - expect(normalizeMemoryDistillPayload({ workspaceId: "ws-1" })).toEqual({ - workspaceId: "ws-1", - }) - }) - - it("rejects missing or blank workspace id", () => { - expect(normalizeMemoryDistillPayload(null)).toBeNull() - expect(normalizeMemoryDistillPayload({})).toBeNull() - expect(normalizeMemoryDistillPayload({ workspaceId: " " })).toBeNull() - }) -}) - -describe("toDistillObservationInput", () => { - it("maps a pending row without inventing fields", () => { - const row = { - id: "obs-1", - workspaceId: "ws-1", - sourceMessageId: "msg-1", - signal: "看重毛利率", - evidenceQuote: "毛利率是核心", - subjectHint: "毛利率", - referencedDocumentIds: ["doc-1", ""], - confidence: 0.9, - status: "pending", - createdAt: new Date("2026-06-30T00:00:00.000Z"), - consumedAt: null, - } as FluidObservation - - expect(toDistillObservationInput(row)).toEqual({ - id: "obs-1", - signal: "看重毛利率", - evidenceQuote: "毛利率是核心", - subjectHint: "毛利率", - confidence: 0.9, - referencedDocumentIds: ["doc-1"], - }) - }) -}) diff --git a/src/domains/memory/distill-workflow.ts b/src/domains/memory/distill-workflow.ts deleted file mode 100644 index c491a718..00000000 --- a/src/domains/memory/distill-workflow.ts +++ /dev/null @@ -1,231 +0,0 @@ -import "server-only" - -import type { WorkflowContext } from "@upstash/workflow" - -import { - DISTILL_BATCH_MAX, - DISTILL_CONSUMED_RETENTION_MS, - DISTILL_DEDUP_CANDIDATES_PER_KIND, -} from "./distill-config" -import { distillMemoryPass } from "./distill-model" -import { - summarizePayloadForContext, - type DistillObservationInput, - type DistillPassKind, - type ExistingMemoryContextItem, - type MemoryOperations, -} from "./distill-types" -import { resolveMemoryOperations } from "./resolve-operations" -import { tokenizeMemoryText } from "./search-index" -import { memoryService } from "./service" -import type { FluidMemoryKind } from "./types" -import { isFluidMemoryKind } from "./types" -import type { FluidMemoryItem, FluidObservation } from "@/infrastructure/db/schema" -import { logger } from "@/lib/logger" - -export type MemoryDistillPayload = { - readonly workspaceId: string -} - -type MemoryDistillWorkflowContext = Pick< - WorkflowContext, - "run" -> - -const PASS_KINDS: readonly { - readonly pass: DistillPassKind - readonly kinds: readonly FluidMemoryKind[] -}[] = [ - { pass: "indicator", kinds: ["indicator_pref"] }, - { pass: "experience", kinds: ["stance", "decision_rule"] }, - { pass: "entity", kinds: ["entity_of_interest"] }, -] - -export function normalizeMemoryDistillPayload( - raw: unknown, -): MemoryDistillPayload | null { - if (!raw || typeof raw !== "object") return null - const workspaceId = getNonEmptyString( - (raw as Record).workspaceId, - ) - if (!workspaceId) return null - return { workspaceId } -} - -/** - * Periodic distill: pending observations → three typed passes → resolve → - * write fluid_memory_items and mark the batch consumed. Capture never writes - * the permanent layer; this job is the only writer. - */ -export async function runMemoryDistillWorkflow(input: { - readonly context: MemoryDistillWorkflowContext - readonly payload: MemoryDistillPayload -}): Promise { - const { context, payload } = input - - const batch = await context.run("select-batch", () => - memoryService.listPendingObservations( - payload.workspaceId, - DISTILL_BATCH_MAX, - ), - ) - if (batch.length === 0) { - logger.info("memory: distill skipped — no pending observations", { - workspaceId: payload.workspaceId, - }) - return - } - - const observationInputs = batch.map(toDistillObservationInput) - const referencedDocumentIds = unionDocumentIds(batch) - const queryTokens = tokenizeBatch(observationInputs) - - const candidatesByKind = await context.run("load-candidates", async () => { - const result: Partial> = {} - for (const kind of [ - "indicator_pref", - "stance", - "decision_rule", - "entity_of_interest", - ] as const) { - result[kind] = await memoryService.findDedupCandidates( - payload.workspaceId, - kind, - queryTokens, - DISTILL_DEDUP_CANDIDATES_PER_KIND, - ) - } - return result - }) - - const passOperations: MemoryOperations[] = [] - for (const { pass, kinds } of PASS_KINDS) { - const existingItems = kinds.flatMap((kind) => - (candidatesByKind[kind] ?? []).flatMap((item) => { - const mapped = toExistingMemoryContextItem(item) - return mapped ? [mapped] : [] - }), - ) - const operations = await context.run(`distill-${pass}`, () => - distillMemoryPass({ - pass, - workspaceId: payload.workspaceId, - observations: observationInputs, - existingItems, - referencedDocumentIds, - }), - ) - // Null = model failure for this pass only; other passes still apply. - if (operations) passOperations.push(operations) - } - - if (passOperations.length === 0) { - logger.warn( - "memory: distill aborted — all passes failed; batch left pending", - { - workspaceId: payload.workspaceId, - batchSize: batch.length, - }, - ) - return - } - - const existingItemRefs = Object.values(candidatesByKind) - .flat() - .map((item) => ({ - id: item.id, - kind: item.kind, - status: item.status, - payload: item.payload, - })) - - const resolved = passOperations.flatMap((operations) => - resolveMemoryOperations({ - operations, - existingItems: existingItemRefs, - referencedDocumentIds, - }), - ) - - const applied = await context.run("apply-and-consume", () => - memoryService.applyDistillBatch({ - workspaceId: payload.workspaceId, - sourceMessageId: null, - operations: resolved, - observationIds: batch.map((row) => row.id), - }), - ) - - const deleted = await context.run("retention", () => - memoryService.deleteExpiredConsumedObservations( - new Date(Date.now() - DISTILL_CONSUMED_RETENTION_MS), - ), - ) - - logger.info("memory: distill workflow finished", { - workspaceId: payload.workspaceId, - batchSize: batch.length, - resolvedCount: resolved.length, - diffCount: applied.diffs.length, - consumedCount: applied.consumedCount, - retentionDeleted: deleted, - }) -} - -export function toDistillObservationInput( - row: FluidObservation, -): DistillObservationInput { - const documentIds = Array.isArray(row.referencedDocumentIds) - ? row.referencedDocumentIds.filter( - (id): id is string => typeof id === "string" && id.length > 0, - ) - : [] - return { - id: row.id, - signal: row.signal, - evidenceQuote: row.evidenceQuote, - subjectHint: row.subjectHint, - confidence: row.confidence, - referencedDocumentIds: documentIds, - } -} - -function toExistingMemoryContextItem( - item: FluidMemoryItem, -): ExistingMemoryContextItem | null { - if (!isFluidMemoryKind(item.kind)) return null - return { - id: item.id, - kind: item.kind, - abstractL0: item.abstractL0, - payloadSummary: summarizePayloadForContext(item.kind, item.payload), - } -} - -function tokenizeBatch( - observations: readonly DistillObservationInput[], -): string[] { - const text = observations - .map((observation) => - [observation.signal, observation.subjectHint ?? "", observation.evidenceQuote] - .filter((part) => part.length > 0) - .join(" "), - ) - .join(" ") - return tokenizeMemoryText(text).map((entry) => entry.token) -} - -function unionDocumentIds(rows: readonly FluidObservation[]): string[] { - const ids = new Set() - for (const row of rows) { - if (!Array.isArray(row.referencedDocumentIds)) continue - for (const id of row.referencedDocumentIds) { - if (typeof id === "string" && id.length > 0) ids.add(id) - } - } - return [...ids] -} - -function getNonEmptyString(value: unknown): string | null { - return typeof value === "string" && value.trim().length > 0 ? value : null -} diff --git a/src/domains/memory/extract-trigger.ts b/src/domains/memory/extract-trigger.ts deleted file mode 100644 index 35b64675..00000000 --- a/src/domains/memory/extract-trigger.ts +++ /dev/null @@ -1,43 +0,0 @@ -import "server-only" - -import { Client } from "@upstash/workflow" - -import type { MemoryExtractPayload } from "./extract-workflow" -import { logger } from "@/lib/logger" - -function resolveBaseURL(): string { - return process.env.NOTEBOOK_PUBLIC_URL ?? "http://localhost:3000" -} - -/** - * Fire-and-forget trigger for the post-turn fluid-memory extraction - * workflow. Each turn is uniquely keyed by its message ids, so unlike the - * source reconcile trigger no cooldown/dedup guard is needed; QStash - * retries cover transient delivery failures. - */ -export async function triggerMemoryExtraction( - payload: MemoryExtractPayload, -): Promise { - const token = process.env.QSTASH_TOKEN - if (!token) { - logger.warn("memory: skipping extraction — QSTASH_TOKEN not set", { - workspaceId: payload.workspaceId, - assistantMessageId: payload.assistantMessageId, - }) - return - } - - try { - await new Client({ token }).trigger({ - url: `${resolveBaseURL()}/api/memory/extract`, - body: payload, - retries: 3, - }) - } catch (error) { - logger.error("memory: failed to trigger extraction workflow", { - workspaceId: payload.workspaceId, - assistantMessageId: payload.assistantMessageId, - message: error instanceof Error ? error.message : String(error), - }) - } -} diff --git a/src/domains/memory/extract-workflow.test.ts b/src/domains/memory/extract-workflow.test.ts deleted file mode 100644 index 41205a55..00000000 --- a/src/domains/memory/extract-workflow.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { normalizeMemoryExtractPayload } from "./extract-workflow" - -describe("normalizeMemoryExtractPayload", () => { - it("accepts a complete payload", () => { - expect( - normalizeMemoryExtractPayload({ - workspaceId: "ws-1", - threadId: "th-1", - userMessageId: "u-1", - assistantMessageId: "a-1", - }), - ).toEqual({ - workspaceId: "ws-1", - threadId: "th-1", - userMessageId: "u-1", - assistantMessageId: "a-1", - }) - }) - - it("rejects missing or blank fields", () => { - expect(normalizeMemoryExtractPayload(null)).toBeNull() - expect( - normalizeMemoryExtractPayload({ - workspaceId: "ws-1", - threadId: "th-1", - userMessageId: "u-1", - }), - ).toBeNull() - expect( - normalizeMemoryExtractPayload({ - workspaceId: " ", - threadId: "th-1", - userMessageId: "u-1", - assistantMessageId: "a-1", - }), - ).toBeNull() - }) -}) diff --git a/src/domains/memory/extract-workflow.ts b/src/domains/memory/extract-workflow.ts deleted file mode 100644 index ec0a538b..00000000 --- a/src/domains/memory/extract-workflow.ts +++ /dev/null @@ -1,128 +0,0 @@ -import "server-only" - -import type { WorkflowContext } from "@upstash/workflow" - -import { triggerMemoryDistill } from "./distill-trigger" -import { captureObservations } from "./extraction-model" -import { memoryService } from "./service" -import { chatThreadService } from "@/domains/chat/thread-service" -import { logger } from "@/lib/logger" - -export type MemoryExtractPayload = { - readonly workspaceId: string - readonly threadId: string - readonly userMessageId: string - readonly assistantMessageId: string -} - -type MemoryExtractWorkflowContext = Pick< - WorkflowContext, - "run" -> - -export function normalizeMemoryExtractPayload( - raw: unknown, -): MemoryExtractPayload | null { - if (!raw || typeof raw !== "object") return null - const record = raw as Record - const workspaceId = getNonEmptyString(record.workspaceId) - const threadId = getNonEmptyString(record.threadId) - const userMessageId = getNonEmptyString(record.userMessageId) - const assistantMessageId = getNonEmptyString(record.assistantMessageId) - if (!workspaceId || !threadId || !userMessageId || !assistantMessageId) { - return null - } - return { workspaceId, threadId, userMessageId, assistantMessageId } -} - -/** - * Per-turn coarse capture: load the turn → LLM observations → append-only - * insert into fluid_observations. Never writes fluid_memory_items (distill - * owns that). After persist, maybe-trigger distill when pending is high enough. - */ -export async function runMemoryExtractWorkflow(input: { - readonly context: MemoryExtractWorkflowContext - readonly payload: MemoryExtractPayload -}): Promise { - const { context, payload } = input - - const turn = await context.run("load-turn", async () => { - const messages = await chatThreadService.listMessages( - payload.workspaceId, - payload.threadId, - ) - const userMessage = messages?.find( - (message) => message.id === payload.userMessageId, - ) - const assistantMessage = messages?.find( - (message) => message.id === payload.assistantMessageId, - ) - if (!userMessage || !assistantMessage) return null - return { - userText: userMessage.content, - assistantText: assistantMessage.content, - referencedDocumentIds: collectCitationDocumentIds( - assistantMessage.citations, - ), - } - }) - if (!turn) { - logger.warn("memory: capture skipped — turn messages not found", { - workspaceId: payload.workspaceId, - threadId: payload.threadId, - }) - return - } - - const observations = await context.run("capture", () => - captureObservations({ - workspaceId: payload.workspaceId, - userText: turn.userText, - assistantText: turn.assistantText, - referencedDocumentIds: turn.referencedDocumentIds, - }), - ) - if (!observations) return - - const inserted = await context.run("persist-observations", async () => { - if (observations.length === 0) return [] - return memoryService.insertObservations({ - workspaceId: payload.workspaceId, - sourceMessageId: payload.assistantMessageId, - referencedDocumentIds: turn.referencedDocumentIds, - observations, - }) - }) - - await context.run("maybe-trigger-distill", async () => { - await triggerMemoryDistill({ workspaceId: payload.workspaceId }) - }) - - logger.info("memory: capture workflow finished", { - workspaceId: payload.workspaceId, - threadId: payload.threadId, - assistantMessageId: payload.assistantMessageId, - observationCount: inserted.length, - }) -} - -function collectCitationDocumentIds(citations: unknown): string[] { - if (!Array.isArray(citations)) return [] - const ids = new Set() - for (const citation of citations) { - if (!citation || typeof citation !== "object") continue - const source = (citation as Record).source - if (!source || typeof source !== "object") continue - const documentId = (source as Record).documentId - if (typeof documentId === "string" && documentId.length > 0) { - ids.add(documentId) - } - } - return [...ids] -} - -function getNonEmptyString(value: unknown): string | null { - return typeof value === "string" && value.trim().length > 0 - ? value - : null -} diff --git a/src/domains/memory/extraction-model.ts b/src/domains/memory/extraction-model.ts deleted file mode 100644 index e013e36f..00000000 --- a/src/domains/memory/extraction-model.ts +++ /dev/null @@ -1,50 +0,0 @@ -import "server-only" - -import { generateObject } from "ai" - -import { - captureOutputSchema, - type CaptureOutput, - type CapturedObservation, -} from "./observation-types" -import { buildCapturePrompt } from "./prompts" -import { CHAT_MODEL } from "@/lib/ai" -import { summarizeUnknownError } from "@/lib/format-log-value" -import { logger } from "@/lib/logger" - -const MEMORY_EXTRACTION_MODEL = process.env.MEMORY_EXTRACTION_MODEL ?? CHAT_MODEL - -/** - * One structured-output call: conversation turn in, raw observations out. - * Best-effort — this runs as a background job, so a model failure skips the - * turn (logged) instead of degrading through fallbacks; the clue typically - * resurfaces in a later turn. - */ -export async function captureObservations(input: { - readonly workspaceId: string - readonly userText: string - readonly assistantText: string - readonly referencedDocumentIds: readonly string[] -}): Promise { - try { - const response = await generateObject({ - model: MEMORY_EXTRACTION_MODEL, - schema: captureOutputSchema, - messages: [ - { - role: "user", - content: buildCapturePrompt(input), - }, - ], - }) - const output: CaptureOutput = response.object - return output.observations - } catch (error) { - logger.warn("memory: capture model call failed; skipping turn", { - workspaceId: input.workspaceId, - model: MEMORY_EXTRACTION_MODEL, - error: summarizeUnknownError(error), - }) - return null - } -} diff --git a/src/domains/memory/observation-types.test.ts b/src/domains/memory/observation-types.test.ts deleted file mode 100644 index 0a99eb6c..00000000 --- a/src/domains/memory/observation-types.test.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { - captureOutputSchema, - capturedObservationSchema, -} from "./observation-types" - -describe("capturedObservationSchema", () => { - it("accepts a full observation", () => { - const parsed = capturedObservationSchema.parse({ - signal: "看重毛利率", - evidenceQuote: "毛利率是核心", - subjectHint: "毛利率", - confidence: 0.9, - }) - expect(parsed.subjectHint).toBe("毛利率") - expect(parsed).not.toHaveProperty("kindHint") - }) - - it("coerces null subjectHint to undefined (single preprocess)", () => { - const parsed = capturedObservationSchema.parse({ - signal: "长期持有", - evidenceQuote: "我做长期投资", - subjectHint: null, - confidence: 1, - }) - expect(parsed.subjectHint).toBeUndefined() - }) - - it("strips unknown kindHint if the model still emits it", () => { - const parsed = capturedObservationSchema.parse({ - signal: "看重毛利率", - evidenceQuote: "毛利率是核心", - kindHint: "indicator", - confidence: 0.9, - }) - expect(parsed).not.toHaveProperty("kindHint") - }) - - it("rejects empty signal or evidenceQuote", () => { - expect(() => - capturedObservationSchema.parse({ - signal: "", - evidenceQuote: "x", - confidence: 0.5, - }), - ).toThrow() - expect(() => - capturedObservationSchema.parse({ - signal: "x", - evidenceQuote: "", - confidence: 0.5, - }), - ).toThrow() - }) -}) - -describe("captureOutputSchema", () => { - it("defaults missing observations to empty array", () => { - expect(captureOutputSchema.parse({})).toEqual({ observations: [] }) - }) - - it("parses a batch of observations", () => { - const parsed = captureOutputSchema.parse({ - observations: [ - { - signal: "跟踪英伟达", - evidenceQuote: "英伟达一直在跟踪", - subjectHint: "英伟达", - confidence: 0.8, - }, - ], - }) - expect(parsed.observations).toHaveLength(1) - expect(parsed.observations[0]?.subjectHint).toBe("英伟达") - expect(parsed.observations[0]).not.toHaveProperty("kindHint") - }) -}) diff --git a/src/domains/memory/observation-types.ts b/src/domains/memory/observation-types.ts deleted file mode 100644 index 63f75987..00000000 --- a/src/domains/memory/observation-types.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { z } from "zod" - -/** - * Coarse-capture contract for the raw observation layer. - * - * Capture records durable USER clues (what the user cares about) only. - * It does not classify into final memory kinds, does not dedup, and never - * writes `fluid_memory_items`. `subjectHint` is an optional topic anchor - * for later clustering — distill owns authoritative typing and merge. - */ - -/** Single concentrated null→undefined coerce for optional capture fields. */ -function nullToUndefined(value: unknown): unknown { - return value === null ? undefined : value -} - -export const capturedObservationSchema = z.object({ - signal: z - .string() - .min(1) - .describe( - "One durable clue about what the USER cares about, in the user's language.", - ), - evidenceQuote: z - .string() - .min(1) - .describe("Short verbatim snippet from the USER turn that supports signal."), - subjectHint: z.preprocess( - nullToUndefined, - z - .string() - .min(1) - .optional() - .describe( - "Optional short topic anchor (metric name, company, topic). Prefer omit when none.", - ), - ), - confidence: z - .number() - .min(0) - .max(1) - .describe("How explicitly the user stated this (1 = explicit)."), -}) - -export const captureOutputSchema = z.object({ - observations: z.array(capturedObservationSchema).default([]), -}) - -export type CapturedObservation = z.infer -export type CaptureOutput = z.infer diff --git a/src/domains/memory/prompts.test.ts b/src/domains/memory/prompts.test.ts deleted file mode 100644 index 2f7c83f7..00000000 --- a/src/domains/memory/prompts.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { buildCapturePrompt } from "./prompts" - -describe("buildCapturePrompt", () => { - const prompt = buildCapturePrompt({ - userText: "毛利率是核心。", - assistantText: "明白。", - referencedDocumentIds: ["doc-1"], - }) - - it("keeps main instructions domain-agnostic and capture-only", () => { - const main = prompt.slice(0, prompt.indexOf("## Illustrative examples")) - expect(main).not.toMatch(/gross margin|市盈率|\bPE\b|Fed|美联储|NVIDIA|英伟达/i) - expect(main).toContain("RAW OBSERVATIONS") - expect(main).toContain("Do NOT classify observations into those kinds") - expect(main).toContain("Do NOT emit any") - expect(main).toContain("kind / type / category field") - expect(main).toContain("Do NOT invent") - expect(main).toContain("create/merge/deprecate operations") - expect(main).not.toContain("kindHint") - expect(main).not.toContain("EXISTING MEMORIES") - expect(main).not.toContain("indicatorPrefs") - expect(main).not.toContain("decisionRules") - expect(main).toContain("Write every free-text value") - expect(main).toMatch(/same\s+language the USER wrote/) - }) - - it("output schema has no early kind classification field", () => { - const schema = prompt.slice(prompt.indexOf("## Output JSON schema")) - expect(schema).not.toContain("kindHint") - expect(schema).toContain("subjectHint") - expect(schema).toContain("signal") - expect(schema).toContain("evidenceQuote") - expect(schema).toContain("confidence") - }) - - it("keeps illustrative examples in a separate section", () => { - expect(prompt).toContain("## Illustrative examples (finance vertical") - expect(prompt).toContain("not exhaustive, not required vocabulary") - const examples = prompt.slice( - prompt.indexOf("## Illustrative examples"), - prompt.indexOf("## Output JSON schema"), - ) - expect(examples).toContain("finance vertical") - expect(examples).toContain("do not force the conversation into this domain") - }) - - it("injects turn context and referenced docs; no existing-memory block", () => { - expect(prompt).toContain("[user]\n毛利率是核心。") - expect(prompt).toContain("doc-1") - expect(prompt).not.toContain("## EXISTING MEMORIES") - }) -}) diff --git a/src/domains/memory/prompts.ts b/src/domains/memory/prompts.ts deleted file mode 100644 index 2f12965f..00000000 --- a/src/domains/memory/prompts.ts +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Coarse-capture prompt for the raw observation layer. - * - * Capture extracts durable USER points of concern only. It does not classify - * into final memory kinds, does not dedup against existing items, and does not - * emit create/merge/deprecate decisions — those belong to distill. - */ - -/** Structural output shape only — no domain content. */ -const CAPTURE_OUTPUT_SCHEMA_BLOCK = `{ - "observations": [{ - "signal": "string — one durable clue about what the USER cares about", - "evidenceQuote": "string — short verbatim USER snippet supporting signal", - "subjectHint": "string optional — topic anchor (metric name / company / topic)", - "confidence": 0.0 - }] -}` - -/** - * Illustrative only — kept separate so the model does not treat these domain - * phrases as required vocabulary. - */ -const ILLUSTRATIVE_EXAMPLES_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) - -These show shape and judgement only. Capture whatever the user actually said; -do not force the conversation into this domain. - -- User names a recurring evaluation metric → one observation (signal + quote). -- User states a durable judgement frame (e.g. long-horizon) → one observation. -- User states a reusable when → then discipline → one observation. -- User says they actively track a named company → one observation. -- Skip: a one-off factual question about a page/number, small talk, or an - assistant suggestion the user did not endorse.` - -const MAIN_INSTRUCTIONS_BLOCK = `You capture RAW OBSERVATIONS for a user's fluid memory pipeline. - -These are cheap, high-recall clues about what the USER cares about. A later -distill step will decide final kinds (indicator / rule / stance / entity) and -merge them. Do NOT classify observations into those kinds. Do NOT emit any -kind / type / category field. Do NOT deduplicate. Do NOT invent -create/merge/deprecate operations. - -Document facts live elsewhere (crystal memory). Never capture document facts, -retrieved numbers, or page content as observations. - -## What to capture - -From the USER turn only, emit zero or more observations when there is real -evidence of a durable, reusable point of concern: - -- signal — one short clue in the user's language (what to remember later). -- evidenceQuote — a short verbatim snippet from the USER turn that supports it. -- subjectHint — optional short topic anchor (metric name, company, topic). Prefer omit when none. -- confidence — 1 only when the user stated it explicitly. - -## Language - -- Keep this instruction set and enum/field names in English. -- Write every free-text value (signal, evidenceQuote, subjectHint) in the same - language the USER wrote in this turn. Do not translate the user's terms into - English unless the user themselves used English. - -## Judgement - -- Capture only durable, reusable clues about what the USER cares about. -- Skip one-off questions, document facts, small talk, and assistant claims the - user did not endorse. -- Prefer atomic clues: one observation per distinct clue. Do not merge unrelated - ideas into one signal. -- Omit optional fields instead of setting them to null. -- If nothing is worth capturing, return {"observations": []}.` - -export function buildCapturePrompt(input: { - readonly userText: string - readonly assistantText: string - readonly referencedDocumentIds: readonly string[] -}): string { - const documentsBlock = - input.referencedDocumentIds.length === 0 - ? "(no documents referenced in this turn)" - : input.referencedDocumentIds.join(", ") - - return `${MAIN_INSTRUCTIONS_BLOCK} - -${ILLUSTRATIVE_EXAMPLES_BLOCK} - -## Output JSON schema (follow exactly; do not invent fields) - -${CAPTURE_OUTPUT_SCHEMA_BLOCK} - -## REFERENCED DOCUMENT IDS - -${documentsBlock} - -## CONVERSATION TURN - -[user] -${input.userText} - -[assistant] -${input.assistantText}` -} diff --git a/src/domains/memory/repository.ts b/src/domains/memory/repository.ts deleted file mode 100644 index 9b44f871..00000000 --- a/src/domains/memory/repository.ts +++ /dev/null @@ -1,500 +0,0 @@ -import "server-only" - -import { and, asc, count, eq, inArray, lt, sql } from "drizzle-orm" -import { Effect } from "effect" - -import type { CapturedObservation } from "./observation-types" -import { toDiffOperation, type ResolvedMemoryOperation } from "./resolve-operations" -import { buildMemoryItemTokens } from "./search-index" -import type { - FluidMemoryDeactivationReason, - FluidMemoryKind, - FluidMemoryPayload, - MemoryDiffOperation, -} from "./types" -import { DbClient, type Db } from "@/infrastructure/db" -import { - fluidMemoryItems, - fluidMemoryTokens, - fluidObservations, - memoryDiffs, - type FluidMemoryItem, - type FluidObservation, - type NewFluidMemoryToken, -} from "@/infrastructure/db/schema" - -export type InsertObservationsInput = { - readonly workspaceId: string - readonly sourceMessageId: string | null - readonly referencedDocumentIds: readonly string[] - readonly observations: readonly CapturedObservation[] -} - -export type ApplyDistillBatchInput = { - readonly workspaceId: string - readonly sourceMessageId: string | null - readonly operations: readonly ResolvedMemoryOperation[] - readonly observationIds: readonly string[] -} - -type MemoryRepository = { - readonly findDedupCandidatesEffect: ( - workspaceId: string, - kind: FluidMemoryKind, - tokens: readonly string[], - limit: number, - ) => Effect.Effect - readonly insertObservationsEffect: ( - input: InsertObservationsInput, - ) => Effect.Effect - readonly countPendingObservationsEffect: ( - workspaceId: string, - ) => Effect.Effect - readonly listPendingObservationsEffect: ( - workspaceId: string, - limit: number, - ) => Effect.Effect - readonly applyDistillBatchEffect: ( - input: ApplyDistillBatchInput, - ) => Effect.Effect< - { - readonly diffs: readonly MemoryDiffOperation[] - readonly consumedCount: number - }, - never, - DbClient - > - readonly deleteExpiredConsumedObservationsEffect: ( - olderThan: Date, - ) => Effect.Effect - readonly listActiveItemsEffect: ( - workspaceId: string, - ) => Effect.Effect< - readonly Pick[], - never, - DbClient - > - /** - * Move a specific set of active items to `inactive` with reason - * `decayed` (the activation-decay job's candidates, already confirmed by - * the caller — this never decides which items on its own). Mirrors the - * distill `deprecate` write path: drops the item's token rows so it stops - * surfacing as a dedup candidate. - */ - readonly deactivateDecayedItemsEffect: ( - workspaceId: string, - itemIds: readonly string[], - ) => Effect.Effect -} - -type RawRowsResult = readonly Row[] | { readonly rows: readonly Row[] } - -type TxClient = Parameters[0]>[0] - -/** - * Retrieve the most lexically-similar active items of one kind, ranked by - * idf-weighted token overlap computed entirely in SQL. Common tokens (high - * document frequency within this workspace + kind) are down-weighted so a - * shared rare term outranks several shared filler characters. - * - * Token rows only exist for active items (see schema invariant), so no - * status filter is needed here. - */ -const findDedupCandidatesEffect: MemoryRepository["findDedupCandidatesEffect"] = - (workspaceId, kind, tokens, limit) => - Effect.gen(function* () { - const db = yield* DbClient - if (tokens.length === 0 || limit <= 0) return [] - - const tokenList = sql.join( - tokens.map((token) => sql`${token}`), - sql`, `, - ) - - const scored = yield* Effect.promise(() => - db.execute<{ itemId: string }>(sql` - SELECT t.item_id AS "itemId", SUM(t.frequency::float8 / df.df) AS score - FROM fluid_memory_tokens t - JOIN ( - SELECT token, COUNT(DISTINCT item_id)::float8 AS df - FROM fluid_memory_tokens - WHERE workspace_id = ${workspaceId}::uuid - AND kind = ${kind} - AND token IN (${tokenList}) - GROUP BY token - ) df ON df.token = t.token - WHERE t.workspace_id = ${workspaceId}::uuid - AND t.kind = ${kind} - AND t.token IN (${tokenList}) - GROUP BY t.item_id - ORDER BY score DESC - LIMIT ${limit} - `), - ) - - const orderedIds = getRawRows(scored).map((row) => row.itemId) - if (orderedIds.length === 0) return [] - - const items = yield* Effect.promise(() => - db - .select() - .from(fluidMemoryItems) - .where(inArray(fluidMemoryItems.id, orderedIds)), - ) - const byId = new Map(items.map((item) => [item.id, item] as const)) - return orderedIds.flatMap((id) => { - const item = byId.get(id) - return item ? [item] : [] - }) - }) - -/** - * Append-only write of coarse-capture clues. Never touches fluid_memory_items. - * Empty input is a no-op (returns []). Status is always `pending`. - */ -const insertObservationsEffect: MemoryRepository["insertObservationsEffect"] = ( - input, -) => - Effect.gen(function* () { - const db = yield* DbClient - if (input.observations.length === 0) return [] - - const documentIds = [...input.referencedDocumentIds] - return yield* Effect.promise(() => - db - .insert(fluidObservations) - .values( - input.observations.map((observation) => ({ - workspaceId: input.workspaceId, - sourceMessageId: input.sourceMessageId, - signal: observation.signal, - evidenceQuote: observation.evidenceQuote, - subjectHint: observation.subjectHint ?? null, - referencedDocumentIds: documentIds, - confidence: observation.confidence, - status: "pending", - })), - ) - .returning(), - ) - }) - -const countPendingObservationsEffect: MemoryRepository["countPendingObservationsEffect"] = - (workspaceId) => - Effect.gen(function* () { - const db = yield* DbClient - const rows = yield* Effect.promise(() => - db - .select({ value: count() }) - .from(fluidObservations) - .where( - and( - eq(fluidObservations.workspaceId, workspaceId), - eq(fluidObservations.status, "pending"), - ), - ), - ) - return Number(rows[0]?.value ?? 0) - }) - -/** - * Oldest-first pending batch for distill. Concurrency across distill runs for - * the same workspace is primarily gated by trigger cooldown + bucketed - * workflowRunId; consume below is conditional on status still being pending. - */ -const listPendingObservationsEffect: MemoryRepository["listPendingObservationsEffect"] = - (workspaceId, limit) => - Effect.gen(function* () { - const db = yield* DbClient - if (limit <= 0) return [] - return yield* Effect.promise(() => - db - .select() - .from(fluidObservations) - .where( - and( - eq(fluidObservations.workspaceId, workspaceId), - eq(fluidObservations.status, "pending"), - ), - ) - .orderBy(asc(fluidObservations.createdAt)) - .limit(limit), - ) - }) - -const applyDistillBatchEffect: MemoryRepository["applyDistillBatchEffect"] = ( - input, -) => - Effect.gen(function* () { - const db = yield* DbClient - return yield* Effect.promise(() => - db.transaction(async (tx) => { - const diffs = await writeOperations( - tx, - input.workspaceId, - input.sourceMessageId, - input.operations, - ) - let consumedCount = 0 - if (input.observationIds.length > 0) { - const consumed = await tx - .update(fluidObservations) - .set({ - status: "consumed", - consumedAt: sql`now()`, - }) - .where( - and( - eq(fluidObservations.workspaceId, input.workspaceId), - eq(fluidObservations.status, "pending"), - inArray(fluidObservations.id, [...input.observationIds]), - ), - ) - .returning({ id: fluidObservations.id }) - consumedCount = consumed.length - } - return { diffs, consumedCount } - }), - ) - }) - -const deleteExpiredConsumedObservationsEffect: MemoryRepository["deleteExpiredConsumedObservationsEffect"] = - (olderThan) => - Effect.gen(function* () { - const db = yield* DbClient - const deleted = yield* Effect.promise(() => - db - .delete(fluidObservations) - .where( - and( - eq(fluidObservations.status, "consumed"), - lt(fluidObservations.createdAt, olderThan), - ), - ) - .returning({ id: fluidObservations.id }), - ) - return deleted.length - }) - -const listActiveItemsEffect: MemoryRepository["listActiveItemsEffect"] = ( - workspaceId, -) => - Effect.gen(function* () { - const db = yield* DbClient - return yield* Effect.promise(() => - db - .select({ - id: fluidMemoryItems.id, - kind: fluidMemoryItems.kind, - createdAt: fluidMemoryItems.createdAt, - }) - .from(fluidMemoryItems) - .where( - and( - eq(fluidMemoryItems.workspaceId, workspaceId), - eq(fluidMemoryItems.status, "active"), - ), - ), - ) - }) - -const deactivateDecayedItemsEffect: MemoryRepository["deactivateDecayedItemsEffect"] = - (workspaceId, itemIds) => - Effect.gen(function* () { - if (itemIds.length === 0) return 0 - - const db = yield* DbClient - return yield* Effect.promise(() => - db.transaction(async (tx) => { - const updated = await tx - .update(fluidMemoryItems) - .set({ - status: "inactive", - deactivationReason: - "decayed" satisfies FluidMemoryDeactivationReason, - updatedAt: sql`now()`, - }) - .where( - and( - eq(fluidMemoryItems.workspaceId, workspaceId), - eq(fluidMemoryItems.status, "active"), - inArray(fluidMemoryItems.id, [...itemIds]), - ), - ) - .returning({ id: fluidMemoryItems.id }) - - if (updated.length > 0) { - await tx.delete(fluidMemoryTokens).where( - inArray( - fluidMemoryTokens.itemId, - updated.map((item) => item.id), - ), - ) - } - return updated.length - }), - ) - }) - -export const memoryRepository: MemoryRepository = { - findDedupCandidatesEffect, - insertObservationsEffect, - countPendingObservationsEffect, - listPendingObservationsEffect, - applyDistillBatchEffect, - deleteExpiredConsumedObservationsEffect, - listActiveItemsEffect, - deactivateDecayedItemsEffect, -} - -async function writeOperations( - tx: TxClient, - workspaceId: string, - sourceMessageId: string | null, - operations: readonly ResolvedMemoryOperation[], -): Promise { - const diffOperations: MemoryDiffOperation[] = [] - - for (const operation of operations) { - switch (operation.op) { - case "create": { - const [inserted] = await tx - .insert(fluidMemoryItems) - .values({ - workspaceId, - kind: operation.kind, - payload: operation.payload, - abstractL0: operation.abstractL0, - overviewL1: operation.overviewL1, - sourceMessageId, - confidence: operation.confidence, - status: "active", - }) - .returning() - if (inserted?.id) { - const tokenRows = tokenRowsFor( - workspaceId, - inserted.id, - operation.kind, - operation.payload, - ) - if (tokenRows.length > 0) { - await tx.insert(fluidMemoryTokens).values(tokenRows) - } - } - diffOperations.push(toDiffOperation(operation, inserted?.id)) - break - } - case "merge": { - const [updated] = await tx - .update(fluidMemoryItems) - .set({ - payload: operation.payload, - abstractL0: operation.abstractL0, - overviewL1: operation.overviewL1, - confidence: operation.confidence, - sourceMessageId, - version: sql`${fluidMemoryItems.version} + 1`, - updatedAt: sql`now()`, - }) - .where( - and( - eq(fluidMemoryItems.id, operation.targetItemId), - eq(fluidMemoryItems.status, "active"), - ), - ) - .returning({ id: fluidMemoryItems.id }) - if (updated) { - await tx - .delete(fluidMemoryTokens) - .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) - const tokenRows = tokenRowsFor( - workspaceId, - operation.targetItemId, - operation.kind, - operation.payload, - ) - if (tokenRows.length > 0) { - await tx.insert(fluidMemoryTokens).values(tokenRows) - } - } - diffOperations.push( - updated - ? toDiffOperation(operation) - : { - op: "skip", - kind: operation.kind, - summary: operation.summary, - reason: "merge target no longer active", - }, - ) - break - } - case "deprecate": { - const [updated] = await tx - .update(fluidMemoryItems) - .set({ - status: "inactive", - deactivationReason: "contradicted" satisfies FluidMemoryDeactivationReason, - updatedAt: sql`now()`, - }) - .where( - and( - eq(fluidMemoryItems.id, operation.targetItemId), - eq(fluidMemoryItems.status, "active"), - ), - ) - .returning({ id: fluidMemoryItems.id }) - if (updated) { - await tx - .delete(fluidMemoryTokens) - .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) - } - diffOperations.push( - updated - ? toDiffOperation(operation) - : { - op: "skip", - kind: operation.kind, - summary: operation.summary, - reason: "deprecate target no longer active", - }, - ) - break - } - case "skip": - diffOperations.push(toDiffOperation(operation)) - break - } - } - - if (diffOperations.length > 0) { - await tx.insert(memoryDiffs).values({ - workspaceId, - sourceMessageId, - operations: [...diffOperations], - }) - } - - return diffOperations -} - -function tokenRowsFor( - workspaceId: string, - itemId: string, - kind: FluidMemoryKind, - payload: FluidMemoryPayload, -): NewFluidMemoryToken[] { - return buildMemoryItemTokens(kind, payload).map((token) => ({ - workspaceId, - itemId, - kind, - token: token.token, - frequency: token.frequency, - })) -} - -function getRawRows(value: RawRowsResult): readonly Row[] { - if (Array.isArray(value)) return value - return (value as { readonly rows: readonly Row[] }).rows -} diff --git a/src/domains/memory/resolve-operations.test.ts b/src/domains/memory/resolve-operations.test.ts deleted file mode 100644 index 03484b9e..00000000 --- a/src/domains/memory/resolve-operations.test.ts +++ /dev/null @@ -1,378 +0,0 @@ -import { describe, expect, it } from "vitest" - -import type { MemoryOperations } from "./resolve-operations" -import { - resolveMemoryOperations, - toDiffOperation, -} from "./resolve-operations" - -const existingItems = [ - { - id: "item-1", - kind: "indicator_pref", - status: "active", - payload: { - name: "毛利率", - aliases: ["gross margin"], - definition: "毛利占营收的比例", - polarity: "higher_better", - importance: "core", - }, - }, - { id: "item-2", kind: "stance", status: "active" }, - { id: "item-3", kind: "stance", status: "inactive" }, - { - id: "item-4", - kind: "entity_of_interest", - status: "active", - payload: { - name: "英伟达", - ticker: "NVDA", - aliases: ["NVIDIA"], - knowhereDocumentIds: ["doc-earlier"], - reason: "用户持续跟踪", - }, - }, -] as const - -function makeOperations( - overrides: Partial = {}, -): MemoryOperations { - return { - indicatorPrefs: [], - stances: [], - decisionRules: [], - entities: [], - ...overrides, - } -} - -function makeIndicatorEntry(decision: { - op: "create" | "skip" | "merge" | "deprecate" - targetItemId?: string - reason?: string -}) { - return { - name: "毛利率", - aliases: ["gross margin"], - definition: "毛利占营收的比例", - polarity: "higher_better" as const, - importance: "core" as const, - abstractL0: "用户看重毛利率", - overviewL1: "用户在分析公司时首先看毛利率。", - confidence: 0.9, - decision, - } -} - -describe("resolveMemoryOperations", () => { - it("passes create through and ignores a stray targetItemId", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - makeIndicatorEntry({ op: "create", targetItemId: "item-1" }), - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved).toHaveLength(1) - expect(resolved[0]).toMatchObject({ - op: "create", - kind: "indicator_pref", - payload: { - name: "毛利率", - aliases: ["gross margin"], - polarity: "higher_better", - }, - }) - expect(resolved[0]).not.toHaveProperty("targetItemId") - }) - - it("merges into an existing active item of the same kind", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - makeIndicatorEntry({ op: "merge", targetItemId: "item-1" }), - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "merge", - kind: "indicator_pref", - targetItemId: "item-1", - }) - }) - - it("downgrades merge to skip when the target is missing", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - makeIndicatorEntry({ op: "merge", targetItemId: "item-999" }), - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "skip", - kind: "indicator_pref", - }) - expect(resolved[0]).not.toHaveProperty("targetItemId") - }) - - it("downgrades merge to skip on kind mismatch", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - makeIndicatorEntry({ op: "merge", targetItemId: "item-2" }), - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]?.op).toBe("skip") - }) - - it("downgrades merge to skip when the target is already inactive", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - stances: [ - { - statement: "长期持有，忽略短期波动", - scope: "投资", - rationale: "用户做长期投资", - abstractL0: "长期投资立场", - overviewL1: "用户强调长期持有。", - confidence: 1, - decision: { op: "merge", targetItemId: "item-3" }, - }, - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]?.op).toBe("skip") - }) - - it("keeps deprecate for an active target", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - stances: [ - { - statement: "美联储短期观点不重要", - scope: "宏观", - rationale: "长期投资", - abstractL0: "不看美联储短期观点", - overviewL1: "用户认为美联储短期观点权重低。", - confidence: 1, - decision: { - op: "deprecate", - targetItemId: "item-2", - reason: "用户改口", - }, - }, - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "deprecate", - targetItemId: "item-2", - reason: "用户改口", - }) - }) - - it("filters entity document ids to the turn's referenced documents", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - entities: [ - { - name: "英伟达", - ticker: "NVDA", - aliases: ["NVIDIA"], - knowhereDocumentIds: ["doc-real", "doc-hallucinated"], - reason: "用户持续跟踪", - abstractL0: "用户关注英伟达", - overviewL1: "用户多次询问英伟达财报。", - confidence: 0.8, - decision: { op: "create" }, - }, - ], - }), - existingItems, - referencedDocumentIds: ["doc-real"], - }) - - expect(resolved[0]).toMatchObject({ - op: "create", - kind: "entity_of_interest", - payload: { - name: "英伟达", - knowhereDocumentIds: ["doc-real"], - }, - }) - }) - - it("unions stored document ids when merging an entity", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - entities: [ - { - name: "英伟达", - ticker: "NVDA", - aliases: ["NVDA Corp"], - knowhereDocumentIds: ["doc-this-turn"], - reason: "用户持续跟踪", - abstractL0: "用户关注英伟达", - overviewL1: "用户多次询问英伟达财报。", - confidence: 0.9, - decision: { op: "merge", targetItemId: "item-4" }, - }, - ], - }), - existingItems, - referencedDocumentIds: ["doc-this-turn"], - }) - - expect(resolved[0]).toMatchObject({ - op: "merge", - targetItemId: "item-4", - payload: { - aliases: ["NVIDIA", "NVDA Corp"], - knowhereDocumentIds: ["doc-earlier", "doc-this-turn"], - }, - }) - }) - - it("unions aliases when merging an indicator preference", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - { - name: "毛利率", - aliases: ["同比毛利率"], - definition: "毛利占营收的比例，也看同比", - polarity: "higher_better" as const, - importance: "core" as const, - abstractL0: "毛利率也看同比", - overviewL1: "用户补充了同比视角。", - confidence: 1, - decision: { op: "merge", targetItemId: "item-1" }, - }, - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "merge", - targetItemId: "item-1", - payload: { - aliases: ["gross margin", "同比毛利率"], - }, - }) - }) - - it("skips create when the payload has no searchable tokens", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - { - name: "!!!", - aliases: [], - definition: "???", - polarity: "context" as const, - importance: "secondary" as const, - abstractL0: "无效符号", - overviewL1: "无法检索。", - confidence: 0.1, - decision: { op: "create" }, - }, - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "skip", - kind: "indicator_pref", - reason: "payload has no searchable tokens", - }) - }) - - it("skips merge when the merged payload has no searchable tokens", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - stances: [ - { - statement: "!!!", - scope: "???", - rationale: "...", - abstractL0: "无效符号", - overviewL1: "无法检索。", - confidence: 0.1, - decision: { op: "merge", targetItemId: "item-2" }, - }, - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "skip", - kind: "stance", - reason: "merged payload has no searchable tokens", - }) - }) -}) - -describe("toDiffOperation", () => { - it("records create with the inserted item id", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [makeIndicatorEntry({ op: "create" })], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(toDiffOperation(resolved[0]!, "new-id")).toEqual({ - op: "create", - kind: "indicator_pref", - summary: "用户看重毛利率", - itemId: "new-id", - }) - }) - - it("records merge/deprecate with their target item id", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - makeIndicatorEntry({ op: "merge", targetItemId: "item-1" }), - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(toDiffOperation(resolved[0]!)).toEqual({ - op: "merge", - kind: "indicator_pref", - summary: "用户看重毛利率", - itemId: "item-1", - }) - }) -}) diff --git a/src/domains/memory/resolve-operations.ts b/src/domains/memory/resolve-operations.ts deleted file mode 100644 index 833b9553..00000000 --- a/src/domains/memory/resolve-operations.ts +++ /dev/null @@ -1,314 +0,0 @@ -import type { MemoryOperations } from "./distill-types" -import { buildMemoryItemTokens } from "./search-index" -import { - parseFluidMemoryPayload, - type FluidMemoryKind, - type FluidMemoryPayload, - type MemoryDiffOperation, -} from "./types" - -export type { MemoryOperations } from "./distill-types" - -/** - * Pure normalization from raw LLM operations to repository-ready - * operations. The LLM output already passed zod validation; this layer - * enforces the invariants the schema cannot express: - * - merge/deprecate must target an existing active item of the same kind - * (otherwise downgraded to skip — conservative, never fabricates) - * - entity knowhereDocumentIds are intersected with the document ids - * allowed for the batch (the model cannot invent provenance) - * - create ignores any targetItemId the model may have emitted - * - create/merge payloads must yield at least one lexical token, otherwise - * the item could never be retrieved for later dedup - * - merge unions aliases (and entity document ids) with the target so - * prior search terms / provenance are not wiped by a partial rewrite - * - * Used by distill (not by per-turn capture). Capture only writes observations. - */ - -export type ResolvedMemoryOperation = - | { - readonly op: "create" - readonly kind: FluidMemoryKind - readonly payload: FluidMemoryPayload - readonly abstractL0: string - readonly overviewL1: string - readonly confidence: number - readonly summary: string - readonly reason?: string - } - | { - readonly op: "skip" - readonly kind: FluidMemoryKind - readonly summary: string - readonly reason?: string - } - | { - readonly op: "merge" - readonly kind: FluidMemoryKind - readonly targetItemId: string - readonly payload: FluidMemoryPayload - readonly abstractL0: string - readonly overviewL1: string - readonly confidence: number - readonly summary: string - readonly reason?: string - } - | { - readonly op: "deprecate" - readonly kind: FluidMemoryKind - readonly targetItemId: string - readonly summary: string - readonly reason?: string - } - -export type ExistingMemoryItemRef = { - readonly id: string - readonly kind: string - readonly status: string - readonly payload?: unknown -} - -const kindToArrayKey = { - indicator_pref: "indicatorPrefs", - stance: "stances", - decision_rule: "decisionRules", - entity_of_interest: "entities", -} as const - -export function resolveMemoryOperations(input: { - readonly operations: MemoryOperations - readonly existingItems: readonly ExistingMemoryItemRef[] - readonly referencedDocumentIds: readonly string[] -}): ResolvedMemoryOperation[] { - const activeById = new Map( - input.existingItems - .filter((item) => item.status === "active") - .map((item) => [item.id, item] as const), - ) - const allowedDocumentIds = new Set(input.referencedDocumentIds) - - const resolved: ResolvedMemoryOperation[] = [] - - for (const kind of Object.keys(kindToArrayKey) as FluidMemoryKind[]) { - const entries = input.operations[kindToArrayKey[kind]] - - for (const entry of entries) { - const summary = entry.abstractL0 - const reason = entry.decision.reason - - if (entry.decision.op === "skip") { - resolved.push({ op: "skip", kind, summary, ...(reason ? { reason } : {}) }) - continue - } - - if (entry.decision.op === "merge" || entry.decision.op === "deprecate") { - const targetId = entry.decision.targetItemId - const target = targetId ? activeById.get(targetId) : undefined - if (!target || target.kind !== kind) { - resolved.push({ - op: "skip", - kind, - summary, - reason: `${entry.decision.op} target missing, inactive, or kind mismatch`, - }) - continue - } - if (entry.decision.op === "deprecate") { - resolved.push({ - op: "deprecate", - kind, - targetItemId: target.id, - summary, - ...(reason ? { reason } : {}), - }) - continue - } - const mergePayload = toPayload( - kind, - entry as Record, - allowedDocumentIds, - ) - if (!mergePayload) { - resolved.push({ - op: "skip", - kind, - summary, - reason: "merged payload failed validation", - }) - continue - } - const preserved = preserveFieldsOnMerge( - kind, - mergePayload, - target.payload, - ) - if (!isIndexable(kind, preserved)) { - resolved.push({ - op: "skip", - kind, - summary, - reason: "merged payload has no searchable tokens", - }) - continue - } - resolved.push({ - op: "merge", - kind, - targetItemId: target.id, - payload: preserved, - abstractL0: entry.abstractL0, - overviewL1: entry.overviewL1, - confidence: entry.confidence, - summary, - ...(reason ? { reason } : {}), - }) - continue - } - - const createPayload = toPayload( - kind, - entry as Record, - allowedDocumentIds, - ) - if (!createPayload) { - resolved.push({ - op: "skip", - kind, - summary, - reason: "payload failed validation", - }) - continue - } - if (!isIndexable(kind, createPayload)) { - resolved.push({ - op: "skip", - kind, - summary, - reason: "payload has no searchable tokens", - }) - continue - } - resolved.push({ - op: "create", - kind, - payload: createPayload, - abstractL0: entry.abstractL0, - overviewL1: entry.overviewL1, - confidence: entry.confidence, - summary, - ...(reason ? { reason } : {}), - }) - } - } - - return resolved -} - -function toPayload( - kind: FluidMemoryKind, - entry: Record, - allowedDocumentIds: ReadonlySet, -): FluidMemoryPayload | null { - const candidate: Record = { - name: entry.name, - aliases: entry.aliases, - definition: entry.definition, - polarity: entry.polarity, - importance: entry.importance, - formulaHint: entry.formulaHint, - statement: entry.statement, - scope: entry.scope, - rationale: entry.rationale, - when: entry.when, - then: entry.then, - priority: entry.priority, - ticker: entry.ticker, - reason: entry.reason, - knowhereDocumentIds: Array.isArray(entry.knowhereDocumentIds) - ? entry.knowhereDocumentIds.filter( - (id): id is string => - typeof id === "string" && allowedDocumentIds.has(id), - ) - : [], - } - return parseFluidMemoryPayload(kind, candidate) -} - -/** Reject payloads that could never be found again by the token index. */ -function isIndexable(kind: FluidMemoryKind, payload: FluidMemoryPayload): boolean { - return buildMemoryItemTokens(kind, payload).length > 0 -} - -/** - * Merge replaces the stored payload, but the model only sees this batch. - * Union aliases (and entity document ids) with the target so earlier search - * terms / provenance survive a partial rewrite. - */ -function preserveFieldsOnMerge( - kind: FluidMemoryKind, - mergedPayload: FluidMemoryPayload, - existingPayload: unknown, -): FluidMemoryPayload { - const existing = parseFluidMemoryPayload(kind, existingPayload) - if (!existing) return mergedPayload - - if ( - kind === "indicator_pref" && - "aliases" in mergedPayload && - "aliases" in existing - ) { - return { - ...mergedPayload, - aliases: unionStrings(existing.aliases, mergedPayload.aliases), - } - } - - if ( - kind === "entity_of_interest" && - "aliases" in mergedPayload && - "aliases" in existing && - "knowhereDocumentIds" in mergedPayload && - "knowhereDocumentIds" in existing - ) { - return { - ...mergedPayload, - aliases: unionStrings(existing.aliases, mergedPayload.aliases), - knowhereDocumentIds: unionStrings( - existing.knowhereDocumentIds, - mergedPayload.knowhereDocumentIds, - ), - } - } - - return mergedPayload -} - -function unionStrings( - left: readonly string[], - right: readonly string[], -): string[] { - return [...new Set([...left, ...right])] -} - -/** Diff-audit view of a resolved operation (itemId filled after write). */ -export function toDiffOperation( - operation: ResolvedMemoryOperation, - itemId?: string, -): MemoryDiffOperation { - const base = { - kind: operation.kind, - summary: operation.summary, - ...(operation.reason ? { reason: operation.reason } : {}), - } - switch (operation.op) { - case "create": - return { op: "create", ...base, ...(itemId ? { itemId } : {}) } - case "merge": - return { op: "merge", ...base, itemId: operation.targetItemId } - case "deprecate": - return { op: "deprecate", ...base, itemId: operation.targetItemId } - case "skip": - return { op: "skip", ...base } - } -} diff --git a/src/domains/memory/search-index.test.ts b/src/domains/memory/search-index.test.ts deleted file mode 100644 index cbf5d0b3..00000000 --- a/src/domains/memory/search-index.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { - buildMemoryItemTokens, - buildMemorySearchText, - tokenizeMemoryText, -} from "./search-index" - -describe("tokenizeMemoryText", () => { - it("splits CJK into single characters and latin into words", () => { - expect(tokenizeMemoryText("毛利率 PE gross_margin")).toEqual([ - { token: "毛", frequency: 1 }, - { token: "利", frequency: 1 }, - { token: "率", frequency: 1 }, - { token: "pe", frequency: 1 }, - { token: "gross_margin", frequency: 1 }, - ]) - }) - - it("counts repeated tokens", () => { - expect(tokenizeMemoryText("PE pe Pe")).toEqual([ - { token: "pe", frequency: 3 }, - ]) - }) - - it("returns empty for whitespace-only input", () => { - expect(tokenizeMemoryText(" ")).toEqual([]) - }) -}) - -describe("buildMemorySearchText", () => { - it("indexes indicator name, aliases, and definition", () => { - expect( - buildMemorySearchText("indicator_pref", { - name: "毛利率", - aliases: ["gross margin"], - definition: "毛利除以营收", - polarity: "higher_better", - importance: "core", - }), - ).toBe("毛利率 gross margin 毛利除以营收") - }) - - it("indexes entity name, aliases, and ticker without reason", () => { - expect( - buildMemorySearchText("entity_of_interest", { - name: "英伟达", - aliases: ["NVIDIA"], - ticker: "NVDA", - knowhereDocumentIds: ["doc-1"], - reason: "一直在跟踪", - }), - ).toBe("英伟达 NVIDIA NVDA") - }) - - it("indexes stance statement and scope", () => { - expect( - buildMemorySearchText("stance", { - statement: "做长期投资", - scope: "宏观短期观点", - rationale: "美联储短期说法不重要", - }), - ).toBe("做长期投资 宏观短期观点") - }) - - it("indexes decision rule when and then", () => { - expect( - buildMemorySearchText("decision_rule", { - when: "毛利率连续两季下滑", - then: "减仓观望", - priority: "high", - rationale: "用户明确说过", - }), - ).toBe("毛利率连续两季下滑 减仓观望") - }) -}) - -describe("buildMemoryItemTokens", () => { - it("tokenizes the search text of an item", () => { - const tokens = buildMemoryItemTokens("indicator_pref", { - name: "PE", - aliases: [], - definition: "市盈率", - polarity: "context", - importance: "secondary", - }) - expect(tokens.map((token) => token.token)).toEqual([ - "pe", - "市", - "盈", - "率", - ]) - }) -}) diff --git a/src/domains/memory/search-index.ts b/src/domains/memory/search-index.ts deleted file mode 100644 index 84a53e9b..00000000 --- a/src/domains/memory/search-index.ts +++ /dev/null @@ -1,84 +0,0 @@ -import type { - DecisionRulePayload, - EntityOfInterestPayload, - FluidMemoryKind, - FluidMemoryPayload, - IndicatorPreferencePayload, - StancePayload, -} from "./types" - -/** - * Lexical search-index helpers for fluid memory dedup retrieval. - * - * Pure and dependency-free so both the write path (indexing an item) and the - * read path (turning a turn into a query) share one tokenizer. Tokenization - * mirrors Knowhere map-nav: lowercase, then emit single CJK characters and - * `[a-z0-9_]+` runs. This handles Chinese (no whitespace segmentation) and - * Latin/alphanumeric terms without any Postgres extension. - */ - -export type MemoryToken = { - readonly token: string - readonly frequency: number -} - -// Single CJK char OR a run of latin letters / digits / underscore. -const TOKEN_PATTERN = - /[a-z0-9_]+|[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/g - -/** - * Build the text that represents an item for lexical matching. Only the - * fields a user would phrase a query against are included (names, aliases, - * short definitions), not provenance or bookkeeping fields. - */ -export function buildMemorySearchText( - kind: FluidMemoryKind, - payload: FluidMemoryPayload, -): string { - return collectSearchParts(kind, payload) - .filter((part) => part.length > 0) - .join(" ") -} - -/** Tokenize free text into deduped tokens with occurrence counts. */ -export function tokenizeMemoryText(text: string): MemoryToken[] { - const counts = new Map() - const matches = text.toLowerCase().match(TOKEN_PATTERN) - if (!matches) return [] - for (const token of matches) { - counts.set(token, (counts.get(token) ?? 0) + 1) - } - return [...counts].map(([token, frequency]) => ({ token, frequency })) -} - -/** Tokens that index one memory item (search text of its payload). */ -export function buildMemoryItemTokens( - kind: FluidMemoryKind, - payload: FluidMemoryPayload, -): MemoryToken[] { - return tokenizeMemoryText(buildMemorySearchText(kind, payload)) -} - -function collectSearchParts( - kind: FluidMemoryKind, - payload: FluidMemoryPayload, -): readonly string[] { - switch (kind) { - case "indicator_pref": { - const p = payload as IndicatorPreferencePayload - return [p.name, ...p.aliases, p.definition] - } - case "stance": { - const p = payload as StancePayload - return [p.statement, p.scope] - } - case "decision_rule": { - const p = payload as DecisionRulePayload - return [p.when, p.then] - } - case "entity_of_interest": { - const p = payload as EntityOfInterestPayload - return [p.name, ...p.aliases, ...(p.ticker ? [p.ticker] : [])] - } - } -} diff --git a/src/domains/memory/service.ts b/src/domains/memory/service.ts deleted file mode 100644 index f711f98b..00000000 --- a/src/domains/memory/service.ts +++ /dev/null @@ -1,146 +0,0 @@ -import "server-only" - -import { databaseRuntime } from "@/domains/workspace/database-runtime" -import { selectDecayCandidates, type DecayCandidate } from "./decay-candidates" -import { - memoryRepository, - type ApplyDistillBatchInput, - type InsertObservationsInput, -} from "./repository" -import type { FluidMemoryKind, MemoryDiffOperation } from "./types" -import { retrievalActivationService } from "@/domains/retrieval-activation/service" -import type { - FluidMemoryItem, - FluidObservation, -} from "@/infrastructure/db/schema" - -type MemoryService = { - readonly findDedupCandidates: ( - workspaceId: string, - kind: FluidMemoryKind, - tokens: readonly string[], - limit: number, - ) => Promise - readonly insertObservations: ( - input: InsertObservationsInput, - ) => Promise - readonly countPendingObservations: (workspaceId: string) => Promise - readonly listPendingObservations: ( - workspaceId: string, - limit: number, - ) => Promise - readonly applyDistillBatch: (input: ApplyDistillBatchInput) => Promise<{ - readonly diffs: readonly MemoryDiffOperation[] - readonly consumedCount: number - }> - readonly deleteExpiredConsumedObservations: ( - olderThan: Date, - ) => Promise - /** - * Active items whose activation-decay score is below `scoreThreshold`. - * Read-only — does not change any item's status. The caller decides the - * threshold and what to do with the result (see - * `deactivateDecayedItems` to actually move candidates to `inactive`). - */ - readonly listDecayCandidates: ( - workspaceId: string, - options: { readonly now: Date; readonly scoreThreshold: number }, - ) => Promise - readonly deactivateDecayedItems: ( - workspaceId: string, - itemIds: readonly string[], - ) => Promise -} - -const findDedupCandidates: MemoryService["findDedupCandidates"] = ( - workspaceId, - kind, - tokens, - limit, -) => - databaseRuntime.runPromise( - memoryRepository.findDedupCandidatesEffect( - workspaceId, - kind, - tokens, - limit, - ), - ) - -const insertObservations: MemoryService["insertObservations"] = (input) => - databaseRuntime.runPromise(memoryRepository.insertObservationsEffect(input)) - -const countPendingObservations: MemoryService["countPendingObservations"] = ( - workspaceId, -) => - databaseRuntime.runPromise( - memoryRepository.countPendingObservationsEffect(workspaceId), - ) - -const listPendingObservations: MemoryService["listPendingObservations"] = ( - workspaceId, - limit, -) => - databaseRuntime.runPromise( - memoryRepository.listPendingObservationsEffect(workspaceId, limit), - ) - -const applyDistillBatch: MemoryService["applyDistillBatch"] = (input) => - databaseRuntime.runPromise(memoryRepository.applyDistillBatchEffect(input)) - -const deleteExpiredConsumedObservations: MemoryService["deleteExpiredConsumedObservations"] = - (olderThan) => - databaseRuntime.runPromise( - memoryRepository.deleteExpiredConsumedObservationsEffect(olderThan), - ) - -const listDecayCandidates: MemoryService["listDecayCandidates"] = async ( - workspaceId, - options, -) => { - const items = await databaseRuntime.runPromise( - memoryRepository.listActiveItemsEffect(workspaceId), - ) - if (items.length === 0) return [] - - const activations = await retrievalActivationService.getActivations( - workspaceId, - "fluid_memory", - items.map((item) => item.id), - ) - const activationsById = new Map( - activations.map((activation) => [ - activation.unitRef, - { - activationCount: activation.activationCount, - lastActivatedAt: activation.lastActivatedAt, - }, - ]), - ) - - return selectDecayCandidates({ - items, - activationsById, - now: options.now, - scoreThreshold: options.scoreThreshold, - }) -} - -const deactivateDecayedItems: MemoryService["deactivateDecayedItems"] = ( - workspaceId, - itemIds, -) => - databaseRuntime.runPromise( - memoryRepository.deactivateDecayedItemsEffect(workspaceId, itemIds), - ) - -export const memoryService: MemoryService = { - findDedupCandidates, - insertObservations, - countPendingObservations, - listPendingObservations, - applyDistillBatch, - deleteExpiredConsumedObservations, - listDecayCandidates, - deactivateDecayedItems, -} diff --git a/src/domains/memory/types.ts b/src/domains/memory/types.ts deleted file mode 100644 index b9975cbd..00000000 --- a/src/domains/memory/types.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { z } from "zod" - -/** - * Fluid memory type contract. - * - * Four typed payload kinds, extracted from conversation turns. The DB - * stores `payload` as jsonb; these schemas are the validation boundary on - * both write (LLM output) and read (repository decode) paths. - */ - -export const fluidMemoryKinds = [ - "indicator_pref", - "stance", - "decision_rule", - "entity_of_interest", -] as const - -export type FluidMemoryKind = (typeof fluidMemoryKinds)[number] - -export const indicatorPreferencePayloadSchema = z.object({ - name: z.string().min(1), - aliases: z.array(z.string()).default([]), - definition: z.string().min(1), - polarity: z.enum(["higher_better", "lower_better", "context"]), - importance: z.enum(["core", "secondary"]), - formulaHint: z.preprocess( - (value) => (value === null ? undefined : value), - z.string().optional(), - ), -}) - -export const stancePayloadSchema = z.object({ - statement: z.string().min(1), - scope: z.string().min(1), - rationale: z.string().min(1), -}) - -export const decisionRulePayloadSchema = z.object({ - when: z.string().min(1), - then: z.string().min(1), - priority: z.enum(["high", "medium", "low"]), - rationale: z.string().min(1), -}) - -export const entityOfInterestPayloadSchema = z.object({ - name: z.string().min(1), - ticker: z.preprocess( - (value) => (value === null ? undefined : value), - z.string().optional(), - ), - aliases: z.array(z.string()).default([]), - knowhereDocumentIds: z.array(z.string()).default([]), - reason: z.string().min(1), -}) - -export type IndicatorPreferencePayload = z.infer< - typeof indicatorPreferencePayloadSchema -> -export type StancePayload = z.infer -export type DecisionRulePayload = z.infer -export type EntityOfInterestPayload = z.infer< - typeof entityOfInterestPayloadSchema -> - -export type FluidMemoryPayload = - | IndicatorPreferencePayload - | StancePayload - | DecisionRulePayload - | EntityOfInterestPayload - -const payloadSchemas: Record> = { - indicator_pref: indicatorPreferencePayloadSchema, - stance: stancePayloadSchema, - decision_rule: decisionRulePayloadSchema, - entity_of_interest: entityOfInterestPayloadSchema, -} - -export function isFluidMemoryKind(value: unknown): value is FluidMemoryKind { - return ( - typeof value === "string" && - (fluidMemoryKinds as readonly string[]).includes(value) - ) -} - -/** Decode a persisted jsonb payload; returns null when the row is malformed. */ -export function parseFluidMemoryPayload( - kind: FluidMemoryKind, - value: unknown, -): FluidMemoryPayload | null { - const result = payloadSchemas[kind].safeParse(value) - return result.success ? result.data : null -} - -/** - * Why an item left `active` for `inactive`. Orthogonal to `status`: `status` - * says whether the item is retrievable today, `deactivationReason` says - * which mechanism moved it out. - * - contradicted — distill decided a later turn reverses this item - * - decayed — the activation-decay job flagged it as unused past threshold - */ -export const fluidMemoryDeactivationReasons = ["contradicted", "decayed"] as const -export type FluidMemoryDeactivationReason = - (typeof fluidMemoryDeactivationReasons)[number] - -/** One decided operation over the memory set; persisted into memory_diffs. */ -export type MemoryDiffOperation = { - readonly op: "create" | "skip" | "merge" | "deprecate" - readonly kind: FluidMemoryKind - readonly itemId?: string - readonly summary: string - readonly reason?: string -} diff --git a/src/domains/retrieval-activation/decay-score.test.ts b/src/domains/retrieval-activation/decay-score.test.ts deleted file mode 100644 index db239ebd..00000000 --- a/src/domains/retrieval-activation/decay-score.test.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { BASE_HALF_LIFE_DAYS, computeDecayScore } from "./decay-score" - -const NOW = new Date("2026-01-08T00:00:00Z") - -describe("computeDecayScore", () => { - it("scores a freshly anchored, never-activated unit as neutral 0.5", () => { - // freq = sigmoid(log1p(0)) = sigmoid(0) = 0.5; recency at age 0 = 1. - const score = computeDecayScore({ - activationCount: 0, - anchorAt: NOW, - now: NOW, - }) - expect(score).toBeCloseTo(0.5, 10) - }) - - it("halves the neutral score after one base half-life with no activations", () => { - const anchorAt = new Date( - NOW.getTime() - BASE_HALF_LIFE_DAYS * 24 * 60 * 60 * 1000, - ) - const score = computeDecayScore({ activationCount: 0, anchorAt, now: NOW }) - expect(score).toBeCloseTo(0.25, 10) - }) - - it("decays monotonically with age for a fixed activation count", () => { - const dayAgo = new Date(NOW.getTime() - 24 * 60 * 60 * 1000) - const weekAgo = new Date(NOW.getTime() - 7 * 24 * 60 * 60 * 1000) - - const scoreAtDay = computeDecayScore({ - activationCount: 2, - anchorAt: dayAgo, - now: NOW, - }) - const scoreAtWeek = computeDecayScore({ - activationCount: 2, - anchorAt: weekAgo, - now: NOW, - }) - - expect(scoreAtDay).toBeGreaterThan(scoreAtWeek) - }) - - it("scores a higher activation count above a lower one at the same age", () => { - const weekAgo = new Date(NOW.getTime() - 7 * 24 * 60 * 60 * 1000) - - const lowCount = computeDecayScore({ - activationCount: 1, - anchorAt: weekAgo, - now: NOW, - }) - const highCount = computeDecayScore({ - activationCount: 10, - anchorAt: weekAgo, - now: NOW, - }) - - expect(highCount).toBeGreaterThan(lowCount) - }) - - it("resets toward the frequency ceiling immediately after a fresh activation", () => { - // A unit with a long activation history but an activation just now - // should score close to its frequency ceiling, not its pre-reset decay. - const score = computeDecayScore({ - activationCount: 5, - anchorAt: NOW, - now: NOW, - }) - const frequencyCeiling = 1 / (1 + Math.exp(-Math.log1p(5))) - expect(score).toBeCloseTo(frequencyCeiling, 10) - }) - - it("stays within (0, 1) across a range of counts and ages", () => { - const activationCounts = [0, 1, 3, 10, 50] - const ageDaysList = [0, 1, 7, 30, 365] - - for (const activationCount of activationCounts) { - for (const ageDays of ageDaysList) { - const anchorAt = new Date( - NOW.getTime() - ageDays * 24 * 60 * 60 * 1000, - ) - const score = computeDecayScore({ activationCount, anchorAt, now: NOW }) - expect(score).toBeGreaterThan(0) - expect(score).toBeLessThan(1) - } - } - }) - - it("clamps negative age (anchor in the future) to zero elapsed time", () => { - const future = new Date(NOW.getTime() + 24 * 60 * 60 * 1000) - const score = computeDecayScore({ - activationCount: 0, - anchorAt: future, - now: NOW, - }) - expect(score).toBeCloseTo(0.5, 10) - }) -}) diff --git a/src/domains/retrieval-activation/decay-score.ts b/src/domains/retrieval-activation/decay-score.ts deleted file mode 100644 index 0e2ca1ba..00000000 --- a/src/domains/retrieval-activation/decay-score.ts +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Time-decay importance score for a retrievable unit (fluid memory item or - * crystal chunk). Pure function, computed at read time — never persisted — - * matching OpenViking's approach of not storing a score that would need - * migration whenever the formula changes. - * - * Structure copied from OpenViking's `hotness_score` - * (`sigmoid(log1p(activationCount)) × exp(-ln2/halfLife × ageDays)`), see - * `.repos/OpenViking/openviking/retrieve/memory_lifecycle.py`. - * - * `BASE_HALF_LIFE_DAYS` is deliberately 2x OpenViking's own default (7 - * days) — a longer grace period before an unused unit's importance - * meaningfully drops, confirmed against simulated day-counts (see the - * `记忆衰减聚类收尾方案` plan for the numbers this was checked against). - * - * One deviation from OpenViking: the recency half-life grows with - * `activationCount` instead of staying fixed, borrowing MemoryBank's - * (arXiv:2305.10250) intuition that repeated recall makes a memory more - * resistant to forgetting (there, strength `S` is incremented by 1 on every - * recall and used directly as the decay time constant). Here: - * - * effectiveHalfLifeDays = BASE_HALF_LIFE_DAYS * (1 + activationCount) - * - * This does not double-count activationCount with the frequency term: the - * frequency term sets the score's baseline ceiling for a given activation - * count, while the half-life growth slows how fast that ceiling erodes as - * time passes without a new activation. - * - * "Reset then decay" (an activation makes the unit feel fresh again, then - * importance decays again from there) is achieved by the caller advancing - * `anchorAt` to the activation time on every write — this function only - * computes the curve from whatever anchor it is given. - */ - -const MS_PER_DAY = 24 * 60 * 60 * 1000 - -/** 2x OpenViking's DEFAULT_HALF_LIFE_DAYS (7) — see rationale above. */ -export const BASE_HALF_LIFE_DAYS = 14 - -export type DecayScoreInput = { - /** Total times this unit has been cited into an answer. */ - readonly activationCount: number - /** Last activation time, or the unit's creation time if never activated. */ - readonly anchorAt: Date - readonly now: Date -} - -/** Always in (0, 1). */ -export function computeDecayScore(input: DecayScoreInput): number { - const activationCount = Math.max(input.activationCount, 0) - const ageDays = Math.max( - (input.now.getTime() - input.anchorAt.getTime()) / MS_PER_DAY, - 0, - ) - - const frequency = sigmoid(Math.log1p(activationCount)) - const effectiveHalfLifeDays = BASE_HALF_LIFE_DAYS * (1 + activationCount) - const decayRate = Math.LN2 / effectiveHalfLifeDays - const recency = Math.exp(-decayRate * ageDays) - - return frequency * recency -} - -function sigmoid(x: number): number { - return 1 / (1 + Math.exp(-x)) -} diff --git a/src/domains/retrieval-activation/repository.test.ts b/src/domains/retrieval-activation/repository.test.ts deleted file mode 100644 index 267c81e0..00000000 --- a/src/domains/retrieval-activation/repository.test.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest" -import { Effect, Layer } from "effect" - -import { retrievalActivationRepository } from "./repository" -import type { Db } from "@/infrastructure/db" - -type InsertValues = { - readonly workspaceId: string - readonly unitType: string - readonly unitRef: string - readonly activationCount: number -} - -afterEach(() => { - vi.restoreAllMocks() -}) - -async function runWithMockDb(insertValues: InsertValues[]) { - const insertBuilder = { - values: vi.fn((values: InsertValues[]) => { - insertValues.push(...values) - return insertBuilder - }), - onConflictDoUpdate: vi.fn(() => insertBuilder), - returning: vi.fn(async () => - insertValues.map((_, index) => ({ id: `activation_${index}` })), - ), - } - const dbMock = { insert: vi.fn(() => insertBuilder) } - const { DbClient } = await vi.importActual( - "@/infrastructure/db", - ) - const dbLayer = Layer.succeed(DbClient, dbMock as unknown as Db) - return { dbLayer, insertBuilder, dbMock } -} - -describe("retrievalActivationRepository.recordActivationsEffect", () => { - it("does nothing and never touches the db for an empty batch", async () => { - const insertValues: InsertValues[] = [] - const { dbLayer, dbMock } = await runWithMockDb(insertValues) - - const written = await Effect.runPromise( - retrievalActivationRepository - .recordActivationsEffect([]) - .pipe(Effect.provide(dbLayer)), - ) - - expect(written).toBe(0) - expect(dbMock.insert).not.toHaveBeenCalled() - }) - - it("collapses duplicate unit refs into a single upsert row", async () => { - const insertValues: InsertValues[] = [] - const { dbLayer, insertBuilder } = await runWithMockDb(insertValues) - - const written = await Effect.runPromise( - retrievalActivationRepository - .recordActivationsEffect([ - { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "doc:chunk_1" }, - { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "doc:chunk_1" }, - { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "doc:chunk_2" }, - ]) - .pipe(Effect.provide(dbLayer)), - ) - - expect(written).toBe(2) - expect(insertBuilder.values).toHaveBeenCalledOnce() - expect(insertValues).toHaveLength(2) - expect(insertValues.map((row) => row.unitRef).sort()).toEqual([ - "doc:chunk_1", - "doc:chunk_2", - ]) - }) - - it("keeps the same unit ref distinct across different unit types and workspaces", async () => { - const insertValues: InsertValues[] = [] - const { dbLayer } = await runWithMockDb(insertValues) - - await Effect.runPromise( - retrievalActivationRepository - .recordActivationsEffect([ - { workspaceId: "ws_1", unitType: "fluid_memory", unitRef: "item_1" }, - { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "item_1" }, - { workspaceId: "ws_2", unitType: "fluid_memory", unitRef: "item_1" }, - ]) - .pipe(Effect.provide(dbLayer)), - ) - - expect(insertValues).toHaveLength(3) - }) -}) - -describe("retrievalActivationRepository.getActivationsEffect", () => { - it("returns [] without querying the db for an empty unit ref list", async () => { - const selectBuilder = { from: vi.fn(), where: vi.fn() } - const dbMock = { select: vi.fn(() => selectBuilder) } - const { DbClient } = await vi.importActual( - "@/infrastructure/db", - ) - const dbLayer = Layer.succeed(DbClient, dbMock as unknown as Db) - - const result = await Effect.runPromise( - retrievalActivationRepository - .getActivationsEffect("ws_1", "fluid_memory", []) - .pipe(Effect.provide(dbLayer)), - ) - - expect(result).toEqual([]) - expect(dbMock.select).not.toHaveBeenCalled() - }) - - it("returns matching activation rows", async () => { - const rows = [ - { - unitRef: "item_1", - activationCount: 3, - lastActivatedAt: new Date("2026-01-01T00:00:00Z"), - }, - ] - const selectBuilder = { - from: vi.fn(() => selectBuilder), - where: vi.fn(async () => rows), - } - const dbMock = { select: vi.fn(() => selectBuilder) } - const { DbClient } = await vi.importActual( - "@/infrastructure/db", - ) - const dbLayer = Layer.succeed(DbClient, dbMock as unknown as Db) - - const result = await Effect.runPromise( - retrievalActivationRepository - .getActivationsEffect("ws_1", "fluid_memory", ["item_1", "item_2"]) - .pipe(Effect.provide(dbLayer)), - ) - - expect(result).toEqual(rows) - }) -}) diff --git a/src/domains/retrieval-activation/repository.ts b/src/domains/retrieval-activation/repository.ts deleted file mode 100644 index 53732acb..00000000 --- a/src/domains/retrieval-activation/repository.ts +++ /dev/null @@ -1,119 +0,0 @@ -import "server-only" - -import { and, eq, inArray, sql } from "drizzle-orm" -import { Effect } from "effect" - -import type { RetrievalUnitType } from "./types" -import { DbClient } from "@/infrastructure/db" -import { retrievalActivations } from "@/infrastructure/db/schema" - -export type RecordActivationInput = { - readonly workspaceId: string - readonly unitType: RetrievalUnitType - readonly unitRef: string -} - -export type ActivationStats = { - readonly unitRef: string - readonly activationCount: number - readonly lastActivatedAt: Date | null -} - -type RetrievalActivationRepository = { - /** - * Upsert one activation (+1, lastActivatedAt = now) per distinct unit. - * Duplicate (workspaceId, unitType, unitRef) entries within `inputs` are - * collapsed to a single +1 — Postgres rejects a multi-row upsert that - * would touch the same conflict target twice in one statement, and - * "cited twice in one answer" should still only count as one activation - * event for this turn. - */ - readonly recordActivationsEffect: ( - inputs: readonly RecordActivationInput[], - ) => Effect.Effect - /** - * Existing ledger rows for a set of unit refs of one type. Units with no - * row (never activated) are simply absent from the result — the caller - * treats that as activationCount 0. - */ - readonly getActivationsEffect: ( - workspaceId: string, - unitType: RetrievalUnitType, - unitRefs: readonly string[], - ) => Effect.Effect -} - -const recordActivationsEffect: RetrievalActivationRepository["recordActivationsEffect"] = - (inputs) => - Effect.gen(function* () { - const deduped = dedupeInputs(inputs) - if (deduped.length === 0) return 0 - - const db = yield* DbClient - const written = yield* Effect.promise(() => - db - .insert(retrievalActivations) - .values( - deduped.map((input) => ({ - workspaceId: input.workspaceId, - unitType: input.unitType, - unitRef: input.unitRef, - activationCount: 1, - lastActivatedAt: sql`now()`, - })), - ) - .onConflictDoUpdate({ - target: [ - retrievalActivations.workspaceId, - retrievalActivations.unitType, - retrievalActivations.unitRef, - ], - set: { - activationCount: sql`${retrievalActivations.activationCount} + 1`, - lastActivatedAt: sql`now()`, - }, - }) - .returning({ id: retrievalActivations.id }), - ) - return written.length - }) - -const getActivationsEffect: RetrievalActivationRepository["getActivationsEffect"] = - (workspaceId, unitType, unitRefs) => - Effect.gen(function* () { - if (unitRefs.length === 0) return [] - - const db = yield* DbClient - const rows = yield* Effect.promise(() => - db - .select({ - unitRef: retrievalActivations.unitRef, - activationCount: retrievalActivations.activationCount, - lastActivatedAt: retrievalActivations.lastActivatedAt, - }) - .from(retrievalActivations) - .where( - and( - eq(retrievalActivations.workspaceId, workspaceId), - eq(retrievalActivations.unitType, unitType), - inArray(retrievalActivations.unitRef, [...unitRefs]), - ), - ), - ) - return rows - }) - -export const retrievalActivationRepository: RetrievalActivationRepository = { - recordActivationsEffect, - getActivationsEffect, -} - -function dedupeInputs( - inputs: readonly RecordActivationInput[], -): RecordActivationInput[] { - const byKey = new Map() - for (const input of inputs) { - byKey.set(`${input.workspaceId}\u0000${input.unitType}\u0000${input.unitRef}`, input) - } - return [...byKey.values()] -} diff --git a/src/domains/retrieval-activation/service.ts b/src/domains/retrieval-activation/service.ts deleted file mode 100644 index 990a6eba..00000000 --- a/src/domains/retrieval-activation/service.ts +++ /dev/null @@ -1,45 +0,0 @@ -import "server-only" - -import { - retrievalActivationRepository, - type ActivationStats, - type RecordActivationInput, -} from "./repository" -import type { RetrievalUnitType } from "./types" -import { databaseRuntime } from "@/domains/workspace/database-runtime" - -type RetrievalActivationService = { - readonly recordActivations: ( - inputs: readonly RecordActivationInput[], - ) => Promise - readonly getActivations: ( - workspaceId: string, - unitType: RetrievalUnitType, - unitRefs: readonly string[], - ) => Promise -} - -const recordActivations: RetrievalActivationService["recordActivations"] = ( - inputs, -) => - databaseRuntime.runPromise( - retrievalActivationRepository.recordActivationsEffect(inputs), - ) - -const getActivations: RetrievalActivationService["getActivations"] = ( - workspaceId, - unitType, - unitRefs, -) => - databaseRuntime.runPromise( - retrievalActivationRepository.getActivationsEffect( - workspaceId, - unitType, - unitRefs, - ), - ) - -export const retrievalActivationService: RetrievalActivationService = { - recordActivations, - getActivations, -} diff --git a/src/domains/retrieval-activation/types.ts b/src/domains/retrieval-activation/types.ts deleted file mode 100644 index 29e18db1..00000000 --- a/src/domains/retrieval-activation/types.ts +++ /dev/null @@ -1,17 +0,0 @@ -/** - * A "unit" is anything retrieval can surface and an answer can actually - * cite. Today there are two kinds: - * - fluid_memory — a `fluid_memory_items` row, keyed by its id - * - crystal_chunk — a Knowhere chunk, which has no local row; keyed by - * `${documentId}:${chunkId}` (see `toChunkUnitRef`) - */ -export const retrievalUnitTypes = ["fluid_memory", "crystal_chunk"] as const -export type RetrievalUnitType = (typeof retrievalUnitTypes)[number] - -/** Composite key for a crystal_chunk unit ref. */ -export function toChunkUnitRef(input: { - readonly documentId: string - readonly chunkId: string -}): string { - return `${input.documentId}:${input.chunkId}` -} diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index 8963f95d..4e9470f3 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -1,9 +1,7 @@ import { sql } from "drizzle-orm"; import { bigint, - doublePrecision, index, - integer, jsonb, pgTable, text, @@ -340,241 +338,3 @@ export const chatMessages = pgTable( export type ChatMessage = typeof chatMessages.$inferSelect; export type NewChatMessage = typeof chatMessages.$inferInsert; - -/** - * Fluid memory: typed insights extracted from human-AI conversation turns - * (as opposed to "crystal memory", which is the parsed document knowledge - * that stays upstream in Knowhere). - * - * One row per extracted insight. `kind` discriminates the typed `payload` - * (see src/domains/memory/types.ts for the payload contract per kind): - * - indicator_pref — a metric the user cares about (name, aliases, - * polarity, importance) - * - stance — a stated position that shapes judgement - * - decision_rule — a when/then rule over indicators - * - entity_of_interest — a company/topic the user tracks - * - * `abstract_l0` / `overview_l1` are the tiered sidecar summaries (L0 = - * one line for pre-filter/dedup context, L1 = short paragraph for later - * cognition injection). L2 is the payload itself. - * - * Lifecycle: rows start `active`; user revisions deactivate rather than - * delete (conservative merge policy), with `version` bumped on merge. - * - * `status` is `active` | `inactive`. `inactive` is not itself a - * disambiguated state — `deactivation_reason` records why an item left - * `active` (e.g. `contradicted`, when distill decides a new turn reverses - * this item). This keeps the decay/lifecycle axis (`status`) separate from - * the reason axis, so an activation-decay job can later flip items to - * `inactive` with a different reason without inventing a new status value. - * - * `source_message_id` points at the assistant message of the turn the - * insight was extracted from; it is set-null on message deletion because - * the insight outlives any single turn. - */ -export const fluidMemoryItems = pgTable( - "fluid_memory_items", - { - id: uuid("id").primaryKey().defaultRandom(), - workspaceId: uuid("workspace_id") - .notNull() - .references(() => workspaces.id, { onDelete: "cascade" }), - kind: text("kind").notNull(), - payload: jsonb("payload").notNull(), - abstractL0: text("abstract_l0").notNull(), - overviewL1: text("overview_l1").notNull(), - sourceMessageId: uuid("source_message_id").references( - () => chatMessages.id, - { onDelete: "set null" }, - ), - confidence: doublePrecision("confidence").notNull(), - status: text("status").notNull(), - deactivationReason: text("deactivation_reason"), - version: integer("version").notNull().default(1), - createdAt: timestamp("created_at", { withTimezone: true }) - .notNull() - .defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }) - .notNull() - .defaultNow(), - }, - (t) => [ - // Workspace lifecycle scans (active vs inactive). - index("fluid_memory_items_workspace_status_idx").on( - t.workspaceId, - t.status, - ), - index("fluid_memory_items_workspace_kind_idx").on(t.workspaceId, t.kind), - ], -); - -export type FluidMemoryItem = typeof fluidMemoryItems.$inferSelect; -export type NewFluidMemoryItem = typeof fluidMemoryItems.$inferInsert; - -/** - * Lexical inverted index over active fluid memory items, used to retrieve - * dedup candidates at extraction time instead of loading the whole memory - * set into the prompt. One row per (item, token); `frequency` counts token - * occurrences in the item's search text. - * - * Invariant: token rows exist iff the owning item is `active`. Writers keep - * this in sync — create inserts rows, merge replaces them, deprecate deletes - * them — so lookups scan tokens alone (no status join) and never surface an - * inactive item. - * - * Tokenization mirrors Knowhere map-nav: single CJK characters plus - * `[a-z0-9_]+` runs. Scoring is idf-weighted token overlap computed in SQL, - * keeping the mechanism on portable Postgres (no pg_trgm/pgvector). - */ -export const fluidMemoryTokens = pgTable( - "fluid_memory_tokens", - { - id: uuid("id").primaryKey().defaultRandom(), - workspaceId: uuid("workspace_id") - .notNull() - .references(() => workspaces.id, { onDelete: "cascade" }), - itemId: uuid("item_id") - .notNull() - .references(() => fluidMemoryItems.id, { onDelete: "cascade" }), - kind: text("kind").notNull(), - token: text("token").notNull(), - frequency: integer("frequency").notNull().default(1), - }, - (t) => [ - // Lookup: candidate tokens within a workspace + kind scope. - index("fluid_memory_tokens_lookup_idx").on( - t.workspaceId, - t.kind, - t.token, - ), - // Rebuild/delete a single item's rows on merge/deprecate. - index("fluid_memory_tokens_item_idx").on(t.itemId), - ], -); - -export type FluidMemoryToken = typeof fluidMemoryTokens.$inferSelect; -export type NewFluidMemoryToken = typeof fluidMemoryTokens.$inferInsert; - -/** - * Append-only audit of extraction decisions, one row per processed turn. - * `operations` is a JSONB array of { op, kind, itemId?, summary, reason? } - * records (op = create | skip | merge | deprecate), mirroring OpenViking's - * memory_diff.json so memory growth stays observable and reversible. - */ -export const memoryDiffs = pgTable( - "memory_diffs", - { - id: uuid("id").primaryKey().defaultRandom(), - workspaceId: uuid("workspace_id") - .notNull() - .references(() => workspaces.id, { onDelete: "cascade" }), - sourceMessageId: uuid("source_message_id").references( - () => chatMessages.id, - { onDelete: "set null" }, - ), - operations: jsonb("operations").notNull(), - createdAt: timestamp("created_at", { withTimezone: true }) - .notNull() - .defaultNow(), - }, - (t) => [ - index("memory_diffs_workspace_created_idx").on(t.workspaceId, t.createdAt), - ], -); - -export type MemoryDiff = typeof memoryDiffs.$inferSelect; -export type NewMemoryDiff = typeof memoryDiffs.$inferInsert; - -/** - * Append-only raw observation layer for fluid memory (L2 evidence). - * - * Each chat turn may write zero or more pending rows here via cheap capture. - * A later distill job consumes a batch, upserts typed items into - * `fluid_memory_items`, and marks these rows `consumed`. Capture never writes - * the distilled layer; distill is the only writer of permanent memory. - * - * Capture stores points of concern only — no early kind classification. - * `subject_hint` is an optional topic anchor for later clustering; distill - * owns the final kind and merge decision. - */ -export const fluidObservations = pgTable( - "fluid_observations", - { - id: uuid("id").primaryKey().defaultRandom(), - workspaceId: uuid("workspace_id") - .notNull() - .references(() => workspaces.id, { onDelete: "cascade" }), - sourceMessageId: uuid("source_message_id").references( - () => chatMessages.id, - { onDelete: "set null" }, - ), - signal: text("signal").notNull(), - evidenceQuote: text("evidence_quote").notNull(), - subjectHint: text("subject_hint"), - referencedDocumentIds: jsonb("referenced_document_ids") - .$type() - .notNull() - .default(sql`'[]'::jsonb`), - confidence: doublePrecision("confidence").notNull(), - status: text("status").notNull().default("pending"), - createdAt: timestamp("created_at", { withTimezone: true }) - .notNull() - .defaultNow(), - consumedAt: timestamp("consumed_at", { withTimezone: true }), - }, - (t) => [ - // Distill scan: pending rows for a workspace in capture order. - index("fluid_observations_workspace_status_created_idx").on( - t.workspaceId, - t.status, - t.createdAt, - ), - ], -); - -export type FluidObservation = typeof fluidObservations.$inferSelect; -export type NewFluidObservation = typeof fluidObservations.$inferInsert; - -/** - * Unified activation ledger for retrievable units, driving time-decay - * importance (see src/domains/retrieval-activation/decay-score.ts). - * - * A "unit" is anything that can be surfaced by retrieval and actually cited - * into an answer: today `fluid_memory` (a `fluid_memory_items` row, keyed by - * its id) and `crystal_chunk` (a Knowhere chunk, which has no local row — - * keyed by `${documentId}:${chunkId}`, composed at write time). - * - * Only "really used in an answer" writes here (a citation), not "entered - * the candidate pool" — this avoids overcounting recall as usage. - * - * For `crystal_chunk`, `created_at` is this row's first-write time (the - * first time Notebook observed this chunk being cited), not the chunk's - * true ingestion time in Knowhere — that timestamp is not available to - * Notebook. This is a known, deliberate approximation. - */ -export const retrievalActivations = pgTable( - "retrieval_activations", - { - id: uuid("id").primaryKey().defaultRandom(), - workspaceId: uuid("workspace_id") - .notNull() - .references(() => workspaces.id, { onDelete: "cascade" }), - unitType: text("unit_type").notNull(), - unitRef: text("unit_ref").notNull(), - activationCount: integer("activation_count").notNull().default(0), - lastActivatedAt: timestamp("last_activated_at", { withTimezone: true }), - createdAt: timestamp("created_at", { withTimezone: true }) - .notNull() - .defaultNow(), - }, - (t) => [ - uniqueIndex("retrieval_activations_unit_idx").on( - t.workspaceId, - t.unitType, - t.unitRef, - ), - ], -); - -export type RetrievalActivation = typeof retrievalActivations.$inferSelect; -export type NewRetrievalActivation = typeof retrievalActivations.$inferInsert; diff --git a/src/integrations/memento/client.ts b/src/integrations/memento/client.ts new file mode 100644 index 00000000..0b0be90a --- /dev/null +++ b/src/integrations/memento/client.ts @@ -0,0 +1,75 @@ +import "server-only" + +import { + FetchHttpClient, + HttpClient, + HttpClientRequest, +} from "@effect/platform" +import { Effect } from "effect" + +import { summarizeUnknownError } from "@/lib/format-log-value" +import { logger } from "@/lib/logger" +import { readMementoConfig } from "./config" + +export type MementoCaptureInput = { + readonly workspaceId: string + readonly sourceMessageId: string | null + readonly userText: string + readonly assistantText: string + readonly referencedDocumentIds: readonly string[] +} + +export type MementoActivationInput = { + readonly workspaceId: string + readonly unitType: "fluid_memory" | "crystal_chunk" + readonly unitRef: string +} + +export async function captureMemoryTurn( + input: MementoCaptureInput, +): Promise { + try { + await postJson("/turns/capture", input) + } catch (error) { + logger.warn("chat: failed to capture memory turn", { + workspaceId: input.workspaceId, + error: summarizeUnknownError(error), + }) + } +} + +export async function recordActivations( + activations: readonly MementoActivationInput[], +): Promise { + if (activations.length === 0) return + + try { + await postJson("/activations", { activations }) + } catch (error) { + logger.warn("chat: failed to record activations", { + workspaceId: activations[0]?.workspaceId, + count: activations.length, + error: summarizeUnknownError(error), + }) + } +} + +async function postJson(path: string, body: unknown): Promise { + const status = await Effect.runPromise( + Effect.gen(function* () { + const { baseUrl, serviceKey } = readMementoConfig() + const request = yield* HttpClientRequest.post(`${baseUrl}${path}`).pipe( + HttpClientRequest.setHeader( + "Authorization", + `Bearer ${serviceKey}`, + ), + HttpClientRequest.bodyJson(body), + ) + const response = yield* HttpClient.execute(request) + return response.status + }).pipe(Effect.provide(FetchHttpClient.layer)), + ) + if (status < 200 || status >= 300) { + throw new Error(`memento ${path}: HTTP ${status}`) + } +} diff --git a/src/integrations/memento/config.ts b/src/integrations/memento/config.ts new file mode 100644 index 00000000..29eb2343 --- /dev/null +++ b/src/integrations/memento/config.ts @@ -0,0 +1,18 @@ +import "server-only" + +export type MementoConfig = { + readonly baseUrl: string + readonly serviceKey: string +} + +export function readMementoConfig(): MementoConfig { + const baseUrl = process.env.MEMENTO_BASE_URL + const serviceKey = process.env.MEMENTO_SERVICE_KEY + if (!baseUrl) { + throw new Error("MEMENTO_BASE_URL is required.") + } + if (!serviceKey) { + throw new Error("MEMENTO_SERVICE_KEY is required.") + } + return { baseUrl, serviceKey } +} diff --git a/src/integrations/memento/memory-tools.ts b/src/integrations/memento/memory-tools.ts new file mode 100644 index 00000000..052f160f --- /dev/null +++ b/src/integrations/memento/memory-tools.ts @@ -0,0 +1,105 @@ +import "server-only" + +import { Client } from "@modelcontextprotocol/sdk/client/index.js" +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js" + +import type { + MemorySearchItem, + MemorySearchKind, + MemorySearchRequest, + MemorySearchResponse, + MemoryToolRuntime, +} from "@/agent-harness" +import { memorySearchKinds } from "@/agent-harness" +import { readMementoConfig } from "./config" + +type MementoMemoryToolsInput = { + readonly workspaceId: string +} + +export const mementoMemoryTools = { + createRuntime(input: MementoMemoryToolsInput): MemoryToolRuntime { + return { + search: (request) => searchWorkspaceMemory(input.workspaceId, request), + } + }, +} as const + +async function searchWorkspaceMemory( + workspaceId: string, + request: MemorySearchRequest, +): Promise { + const { baseUrl, serviceKey } = readMementoConfig() + const transport = new StreamableHTTPClientTransport(new URL(`${baseUrl}/mcp`), { + requestInit: { + headers: { + Authorization: `Bearer ${serviceKey}`, + }, + }, + }) + const client = new Client({ + name: "knowhere-notebook", + version: "0.1.0", + }) + await client.connect(transport) + try { + const result = await client.callTool({ + name: "memory_search", + arguments: { + workspaceId, + query: request.query, + ...(request.kinds ? { kinds: request.kinds } : {}), + }, + }) + if (result.isError) { + throw new Error("memento memory_search failed") + } + return parseMemorySearchResponse(result.structuredContent) + } finally { + await client.close() + } +} + +function parseMemorySearchResponse(value: unknown): MemorySearchResponse { + if (!value || typeof value !== "object") { + throw new Error("memento memory_search returned no structured content") + } + const record = value as Record + if (typeof record.query !== "string" || !Array.isArray(record.items)) { + throw new Error("memento memory_search structured content is invalid") + } + return { + query: record.query, + items: record.items.map(parseMemorySearchItem), + } +} + +function parseMemorySearchItem(value: unknown): MemorySearchItem { + if (!value || typeof value !== "object") { + throw new Error("memento memory_search item is invalid") + } + const item = value as Record + if ( + typeof item.ref !== "string" || + typeof item.itemId !== "string" || + typeof item.abstractL0 !== "string" || + typeof item.overviewL1 !== "string" || + !isMemorySearchKind(item.kind) + ) { + throw new Error("memento memory_search item is invalid") + } + return { + ref: item.ref, + itemId: item.itemId, + kind: item.kind, + abstractL0: item.abstractL0, + overviewL1: item.overviewL1, + } +} + +function isMemorySearchKind(value: unknown): value is MemorySearchKind { + return ( + typeof value === "string" && + (memorySearchKinds as readonly string[]).includes(value) + ) +} From a5c6b0f22ed236c21bcfb181ff8811d8341a52df Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Thu, 10 Sep 2026 17:41:47 +0800 Subject: [PATCH 09/11] refactor(chat): update namespace handling and improve Knowhere client configuration - Changed the namespace from "notebook-namespace" to "default" in chat service tests to align with new defaults. - Updated the chat service to use a shared library namespace instead of fetching compatible namespaces. - Increased the timeout for the Knowhere client to 120 seconds to enhance reliability during API calls. These changes streamline namespace management and improve the integration with the Knowhere API. --- src/domains/chat/service.test.ts | 4 ++-- src/domains/chat/service.ts | 4 ++-- src/integrations/knowhere.test.ts | 1 + src/integrations/knowhere.ts | 1 + 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/domains/chat/service.test.ts b/src/domains/chat/service.test.ts index 133e0a54..4aa1a5a0 100644 --- a/src/domains/chat/service.test.ts +++ b/src/domains/chat/service.test.ts @@ -54,7 +54,7 @@ describe("handleChatTurn", () => { }); } expect(retrieval.query).toHaveBeenCalledWith({ - namespace: "notebook-namespace", + namespace: "default", query: "What does the document say?", topK: 8, useAgentic: true, @@ -245,7 +245,7 @@ describe("handleChatTurn", () => { knowhereTools: expect.any(Object), }); expect(retrieval.query).toHaveBeenCalledWith({ - namespace: "notebook-namespace", + namespace: "default", query: "Tesla Q4 2025 Update energy generation and storage deployments", topK: 8, useAgentic: true, diff --git a/src/domains/chat/service.ts b/src/domains/chat/service.ts index 23bfa68d..7839ab02 100644 --- a/src/domains/chat/service.ts +++ b/src/domains/chat/service.ts @@ -9,7 +9,7 @@ import { } from "." import { toChatMessageView } from "./view" import type { ChatMessage, ChatThread, Source, Workspace } from "@/infrastructure/db/schema" -import { getCompatibleNamespaces } from "@/domains/sources/namespace" +import { sharedLibraryNamespace } from "@/domains/sources/namespace" import type { ChatArtifactView, ChatCitationView, @@ -125,7 +125,7 @@ export const handleChatTurnEffect = (input: ChatTurnInput) => const answer = yield* answerQuestionWithRetrieval({ question: input.question, namespace: input.workspace.namespace, - namespaces: getCompatibleNamespaces(input.workspace), + namespaces: [sharedLibraryNamespace], sources: readySources, useAgentic: input.useAgentic ?? true, excludedSourceIds: input.excludedSourceIds, diff --git a/src/integrations/knowhere.test.ts b/src/integrations/knowhere.test.ts index 74b8c253..a2c4845d 100644 --- a/src/integrations/knowhere.test.ts +++ b/src/integrations/knowhere.test.ts @@ -58,6 +58,7 @@ describe("makeKnowhereClient", () => { expect(constructorSpy).toHaveBeenCalledWith({ apiKey: "sk_test", baseURL: "https://api-staging.knowhereto.ai", + timeout: 120_000, }); }); diff --git a/src/integrations/knowhere.ts b/src/integrations/knowhere.ts index e55fc0bd..a1ca2ed5 100644 --- a/src/integrations/knowhere.ts +++ b/src/integrations/knowhere.ts @@ -37,6 +37,7 @@ export function makeKnowhereClient(apiKey: string): Knowhere { const options: ConstructorParameters[0] = { apiKey, baseURL: process.env.KNOWHERE_BASE_URL, + timeout: 120_000, } const client = new Knowhere(options) return wrapKnowhereClient(client) From 99cfe1df6fb4a142459eea6907280d2d81c81c28 Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Fri, 11 Sep 2026 13:00:49 +0800 Subject: [PATCH 10/11] refactor(knowhere): streamline response handling and remove unused types - Removed unused types and functions related to KnowledgeReadResponse and KnowledgeGrepResponse from the knowhere-text and ledger modules. - Updated the knowhereToolText to handle chunk formatting with a new chunkPickStart parameter for better citation management. - Adjusted tests to reflect changes in response handling and ensure proper functionality with the updated code structure. This refactor enhances code clarity and maintains focus on relevant response types, improving overall maintainability. --- src/agent-harness/knowhere-text.test.ts | 190 +-------- src/agent-harness/knowhere-text.ts | 162 +------- src/agent-harness/ledger.test.ts | 234 +++-------- src/agent-harness/ledger.ts | 164 +------- src/agent-harness/runtime.test.ts | 151 +++++-- src/agent-harness/runtime.ts | 377 +++++------------- src/agent-harness/types.ts | 34 +- src/components/chat-message-list.tsx | 12 +- src/domains/chat/contracts.ts | 2 - src/domains/chat/index.test.ts | 243 +---------- src/domains/chat/index.ts | 19 +- src/domains/chat/knowhere-tools.ts | 145 +------ src/domains/chat/media-assets.test.ts | 25 ++ src/domains/chat/media-assets.ts | 27 +- src/domains/chat/page-citation-assets.test.ts | 25 ++ src/domains/chat/page-citation-assets.ts | 7 +- src/domains/chat/prompt.ts | 2 +- src/domains/chat/route-answer.ts | 1 - src/domains/chat/service.ts | 2 - 19 files changed, 448 insertions(+), 1374 deletions(-) diff --git a/src/agent-harness/knowhere-text.test.ts b/src/agent-harness/knowhere-text.test.ts index 5ec8abbb..af1094e9 100644 --- a/src/agent-harness/knowhere-text.test.ts +++ b/src/agent-harness/knowhere-text.test.ts @@ -1,10 +1,5 @@ import { describe, expect, it } from "vitest" -import type { - KnowledgeGrepResponse, - KnowledgeOutline, - KnowledgeReadResponse, - RetrievalQueryResponse, -} from "@ontos-ai/knowhere-sdk" +import type { RetrievalQueryResponse } from "@ontos-ai/knowhere-sdk" import { createEvidenceLedger } from "./ledger" import { knowhereToolText } from "./knowhere-text" @@ -18,11 +13,13 @@ describe("knowhereToolText", () => { const text = knowhereToolText.formatSearch({ response, retrievalCount: snapshot.retrievalCount, + chunkPickStart: 0, chunks: snapshot.chunks, assets: snapshot.assets, }) expect(text).toContain('') + expect(text).toContain('pick="1"') expect(text).toContain('ref="r1:result:1"') expect(text).toContain('ref="asset:r1:result:1"') expect(text).toContain("Page one summary.") @@ -33,77 +30,17 @@ describe("knowhereToolText", () => { expect(text).not.toContain("https://assets.example/page-1.png") }) - it("formats list and outline responses", () => { - const listText = knowhereToolText.formatListDocuments({ - documents: [ - { - documentId: "doc_1", - revisionKey: "job_1", - namespace: "notebook", - sourceFileName: "report.pdf", - title: "Report", - status: "ready", - }, - ], - }) - const outlineText = knowhereToolText.formatOutline(makeOutlineResponse()) - - expect(listText).toContain('') - expect(listText).toContain('documentId="doc_1"') - expect(outlineText).toContain( - '', - ) - expect(outlineText).toContain('sectionPath="Root / Revenue"') - }) - - it("formats full read chunk bodies without truncating large content", () => { - const ledger = createEvidenceLedger() - const largeContent = `BEGIN ${"full chunk body ".repeat(700)} END` - const response = makeReadResponse(largeContent) - const snapshot = ledger.addReadChunksResponse(response) - - const text = knowhereToolText.formatReadChunks({ - response, - chunks: snapshot.chunks, - assets: snapshot.assets, - }) - - expect(text).toContain('') - expect(text).toContain('ref="read1:chunk:1"') - expect(text).toContain(largeContent) - expect(text).not.toContain("...[truncated]") - expect(text).not.toContain("https://assets.example/page-1.png") - }) - - it("formats grep matches with continuation metadata", () => { - const ledger = createEvidenceLedger() - const response = makeGrepResponse() - const snapshot = ledger.addGrepChunksResponse(response) - - const text = knowhereToolText.formatGrepChunks({ - response, - chunks: snapshot.chunks, - assets: snapshot.assets, - }) - - expect(text).toContain('') - expect(text).toContain('truncated="true"') - expect(text).toContain('continuationCursor="cursor_2"') - expect(text).toContain('ref="grep1:match:1"') - expect(text).toContain("matched penalty snippet") - }) - it("formats errors as tagged text", () => { expect( knowhereToolText.formatError({ - operation: "read_chunks", - message: "A documentId is required.", + operation: "search", + message: "Knowhere search failed.", }), ).toBe( [ - '', + '', "", - "A documentId is required.", + "Knowhere search failed.", "", "", ].join("\n"), @@ -147,116 +84,3 @@ function makeSearchResponse(): RetrievalQueryResponse { referencedChunks: [], } } - -function makeReadResponse(content: string): KnowledgeReadResponse { - return { - document: { - localDocumentId: "doc_1", - documentId: "doc_1", - jobId: "job_1", - namespace: "notebook", - sourceFileName: "report.pdf", - chunkCount: 1, - typeCounts: { text: 0, image: 0, table: 0, page: 1 }, - resultDirectoryPath: "parsed-storage:doc_1/job_1", - createdAt: new Date("2026-01-01T00:00:00Z"), - updatedAt: new Date("2026-01-01T00:00:00Z"), - }, - chunks: [ - { - position: 1, - chunkId: "chunk_page_1", - chunkType: "page", - contentSource: "content", - content, - readableContent: content, - sectionPath: "Root / Page 1", - sourceChunkPath: "pages/page-1.md", - filePath: "pages/page-1.png", - assetUrl: "https://assets.example/page-1.png", - pageNumbers: [1], - metadata: { - pageNums: [1], - pageAssets: [ - { - pageNum: 1, - artifactRef: "page_citation_assets/page-1.png", - assetUrl: "https://assets.example/page-1.png", - contentType: "image/png", - }, - ], - }, - }, - ], - page: 1, - pageSize: 1, - totalChunks: 1, - totalPages: 1, - } -} - -function makeOutlineResponse(): KnowledgeOutline { - return { - document: { - localDocumentId: "doc_1", - documentId: "doc_1", - jobId: "job_1", - namespace: "notebook", - sourceFileName: "report.pdf", - chunkCount: 1, - typeCounts: { text: 1, image: 0, table: 0, page: 0 }, - resultDirectoryPath: "parsed-storage:doc_1/job_1", - createdAt: new Date("2026-01-01T00:00:00Z"), - updatedAt: new Date("2026-01-01T00:00:00Z"), - }, - totalChunks: 1, - typeCounts: { text: 1, image: 0, table: 0, page: 0 }, - sections: [ - { - sectionPath: "Root / Revenue", - sectionTitle: "Revenue", - sectionLevel: 2, - summary: "Revenue summary.", - startChunk: 1, - endChunk: 1, - chunkCount: 1, - typeCounts: { text: 1, image: 0, table: 0, page: 0 }, - children: [], - }, - ], - sectionTree: [], - } -} - -function makeGrepResponse(): KnowledgeGrepResponse { - return { - document: { - localDocumentId: "doc_1", - documentId: "doc_1", - jobId: "job_1", - namespace: "notebook", - sourceFileName: "contract.pdf", - chunkCount: 4, - typeCounts: { text: 4, image: 0, table: 0, page: 0 }, - resultDirectoryPath: "parsed-storage:doc_1/job_1", - createdAt: new Date("2026-01-01T00:00:00Z"), - updatedAt: new Date("2026-01-01T00:00:00Z"), - }, - matches: [ - { - position: 3, - chunkId: "chunk_3", - chunkType: "text", - sectionPath: "Root / Penalties", - sourceChunkPath: "chunks/chunk-3.md", - filePath: "contract.pdf", - startOffset: 10, - endOffset: 17, - snippet: "matched penalty snippet", - }, - ], - scannedChunks: 4, - truncated: true, - continuationCursor: "cursor_2", - } -} diff --git a/src/agent-harness/knowhere-text.ts b/src/agent-harness/knowhere-text.ts index 2140e7ec..5f55ff78 100644 --- a/src/agent-harness/knowhere-text.ts +++ b/src/agent-harness/knowhere-text.ts @@ -1,15 +1,6 @@ -import type { - KnowledgeGrepResponse, - KnowledgeOutline, - KnowledgeReadResponse, - RetrievalQueryResponse, -} from "@ontos-ai/knowhere-sdk" +import type { RetrievalQueryResponse } from "@ontos-ai/knowhere-sdk" -import type { - EvidenceAsset, - EvidenceChunk, - KnowhereListDocumentsResponse, -} from "./types" +import type { EvidenceAsset, EvidenceChunk } from "./types" type EvidenceDelta = { readonly chunks: readonly EvidenceChunk[] @@ -19,14 +10,7 @@ type EvidenceDelta = { type SearchTextInput = EvidenceDelta & { readonly response: RetrievalQueryResponse readonly retrievalCount: number -} - -type ReadChunksTextInput = EvidenceDelta & { - readonly response: KnowledgeReadResponse -} - -type GrepChunksTextInput = EvidenceDelta & { - readonly response: KnowledgeGrepResponse + readonly chunkPickStart: number } type ErrorTextInput = { @@ -34,12 +18,7 @@ type ErrorTextInput = { readonly message: string } -type KnowhereOperation = - | "search" - | "list_documents" - | "get_document_outline" - | "read_chunks" - | "grep_chunks" +type KnowhereOperation = "search" const assetInstruction = "Notebook returned image/page asset refs. Call inspectImage with the asset refs you will cite before finalize so OCR/visual context and provenance boxes exist. Do not expose raw asset URLs." @@ -58,99 +37,12 @@ export const knowhereToolText = { }), // Model grounding comes from (results). Do not also inject // evidenceText — same bodies, no citeable refs, doubles context. - formatEvidenceChunks(input.chunks), + formatEvidenceChunks(input.chunks, input.chunkPickStart), formatEvidenceAssets(input.assets), formatAssetInstruction(input.assets), ]) }, - formatListDocuments(response: KnowhereListDocumentsResponse): string { - return wrapKnowhereBlock("list_documents", [ - formatTag("summary", { documentCount: String(response.documents.length) }), - ...response.documents.map((document, index) => - formatSelfClosingTag("document", { - index: String(index + 1), - documentId: document.documentId, - localDocumentId: document.localDocumentId, - revisionKey: document.revisionKey, - namespace: document.namespace, - sourceFileName: document.sourceFileName, - title: document.title, - status: document.status, - chunkCount: - typeof document.chunkCount === "number" - ? String(document.chunkCount) - : undefined, - }), - ), - ]) - }, - - formatOutline(response: KnowledgeOutline): string { - return wrapKnowhereBlock("get_document_outline", [ - formatTag("document", { - documentId: response.document.documentId, - localDocumentId: response.document.localDocumentId, - revisionKey: response.document.jobId, - sourceFileName: response.document.sourceFileName, - totalChunks: String(response.totalChunks), - truncated: response.truncated === true ? "true" : undefined, - continuationCursor: response.continuationCursor, - }), - ...response.sections.map((section) => formatSection(section, 0)), - ]) - }, - - formatReadChunks(input: ReadChunksTextInput): string { - return wrapKnowhereBlock("read_chunks", [ - formatTag("document", { - documentId: input.response.document.documentId, - localDocumentId: input.response.document.localDocumentId, - revisionKey: input.response.document.jobId, - sourceFileName: input.response.document.sourceFileName, - page: - typeof input.response.page === "number" - ? String(input.response.page) - : undefined, - pageSize: - typeof input.response.pageSize === "number" - ? String(input.response.pageSize) - : undefined, - totalChunks: - typeof input.response.totalChunks === "number" - ? String(input.response.totalChunks) - : undefined, - totalPages: - typeof input.response.totalPages === "number" - ? String(input.response.totalPages) - : undefined, - nextChunk: - typeof input.response.nextChunk === "number" - ? String(input.response.nextChunk) - : undefined, - }), - formatEvidenceChunks(input.chunks), - formatEvidenceAssets(input.assets), - formatAssetInstruction(input.assets), - ]) - }, - - formatGrepChunks(input: GrepChunksTextInput): string { - return wrapKnowhereBlock("grep_chunks", [ - formatTag("document", { - documentId: input.response.document.documentId, - localDocumentId: input.response.document.localDocumentId, - revisionKey: input.response.document.jobId, - sourceFileName: input.response.document.sourceFileName, - matchCount: String(input.response.matches.length), - scannedChunks: String(input.response.scannedChunks), - truncated: input.response.truncated ? "true" : "false", - continuationCursor: input.response.continuationCursor, - }), - formatEvidenceChunks(input.chunks), - ]) - }, - formatError(input: ErrorTextInput): string { return [ formatOpenTag("knowhere", { @@ -174,14 +66,18 @@ function wrapKnowhereBlock( ].join("\n") } -function formatEvidenceChunks(chunks: readonly EvidenceChunk[]): string { +function formatEvidenceChunks( + chunks: readonly EvidenceChunk[], + chunkPickStart: number, +): string { if (chunks.length === 0) return "" return [ "", - ...chunks.map((chunk) => + ...chunks.map((chunk, index) => [ formatOpenTag("chunk", { + pick: String(chunkPickStart + index + 1), ref: chunk.ref, kind: chunk.kind, chunkId: chunk.chunkId, @@ -228,42 +124,6 @@ function formatAssetInstruction(assets: readonly EvidenceAsset[]): string { return formatTextTag("asset_instruction", assetInstruction) } -function formatSection( - section: KnowledgeOutline["sections"][number], - depth: number, -): string { - return [ - formatOpenTag("section", { - depth: String(depth), - sectionPath: section.sectionPath, - sectionTitle: section.sectionTitle, - sectionLevel: String(section.sectionLevel), - startChunk: - typeof section.startChunk === "number" - ? String(section.startChunk) - : undefined, - endChunk: - typeof section.endChunk === "number" - ? String(section.endChunk) - : undefined, - chunkCount: String(section.chunkCount), - }), - formatOptionalTextTag("summary", section.summary), - ...section.children.map((child) => formatSection(child, depth + 1)), - "", - ] - .filter((part) => part.trim().length > 0) - .join("\n") -} - -function formatOptionalTextTag( - tagName: string, - value: string | null | undefined, -): string { - const trimmedValue = value?.trim() - return trimmedValue ? formatTextTag(tagName, trimmedValue) : "" -} - function formatTextTag(tagName: string, value: string): string { return [`<${tagName}>`, value, ``].join("\n") } diff --git a/src/agent-harness/ledger.test.ts b/src/agent-harness/ledger.test.ts index 3af12759..6d3bdb32 100644 --- a/src/agent-harness/ledger.test.ts +++ b/src/agent-harness/ledger.test.ts @@ -1,9 +1,5 @@ import { describe, expect, it } from "vitest" -import type { - KnowledgeGrepResponse, - KnowledgeReadResponse, - RetrievalQueryResponse, -} from "@ontos-ai/knowhere-sdk" +import type { RetrievalQueryResponse } from "@ontos-ai/knowhere-sdk" import { createEvidenceLedger } from "./ledger" @@ -117,118 +113,80 @@ describe("createEvidenceLedger", () => { ) }) - it("adds read chunk refs and page image assets", () => { + it("does not throw when a result is missing chunkType", () => { + // Knowhere's chunkType is declared as a required string in the SDK + // type, but real retrieval results can omit it at runtime, same as the + // referencedChunks case above. Unlike referencedChunks (which carry no + // real content), results carry real content, so the chunk must still be + // kept in the ledger -- only asset-type detection should be guarded. const ledger = createEvidenceLedger() - const snapshot = ledger.addReadChunksResponse(makeReadResponse()) - - expect(snapshot.chunks).toEqual([ - expect.objectContaining({ - ref: "read1:chunk:1", - kind: "read_chunk", - content: "Full page content.", - contentPreview: "Full page content.", - assetRef: "asset:read1:chunk:1", - }), - ]) - expect(snapshot.assets).toEqual([ - expect.objectContaining({ - ref: "asset:read1:chunk:1", - chunkRef: "read1:chunk:1", - type: "image", - sourcePath: "page_citation_assets/page-1.png", - }), - ]) - }) - - it("adds grep match refs", () => { - const ledger = createEvidenceLedger() - - const snapshot = ledger.addGrepChunksResponse(makeGrepResponse()) - - expect(snapshot.chunks).toEqual([ - expect.objectContaining({ - ref: "grep1:match:1", - kind: "grep_match", - chunkId: "chunk_3", - content: "matched penalty snippet", - source: expect.objectContaining({ - documentId: "doc_contract", - sourceFileName: "contract.pdf", - sectionPath: "Root / Penalties", - }), - }), - ]) - }) - - it("copies page metadata onto grep matches from the same chunk id", () => { - const ledger = createEvidenceLedger() - ledger.addReadChunksResponse(makeReadResponse()) - - const snapshot = ledger.addGrepChunksResponse({ - ...makeGrepResponse(), - matches: [ + const snapshot = ledger.addRetrievalResponse({ + namespace: "default", + query: "hypertension target", + routerUsed: "agent_explore", + answerText: "", + evidenceText: "[E1] some evidence", + stopReason: "finished", + failureReason: null, + results: [ { - position: 1, - chunkId: "chunk_page_1", - chunkType: "page", - sectionPath: "Root / Page 1", - sourceChunkPath: "pages/page-1.md", - startOffset: 0, - endOffset: 12, - snippet: "Full page snip", + content: "Target BP is <130/80 mmHg.", + chunkType: undefined as unknown as string, + score: 0.9, + assetUrl: "https://assets.example/images/chart.png", + source: { + documentId: "doc_1", + sourceFileName: "guideline.pdf", + sectionPath: "BP targets", + }, }, ], + referencedChunks: [], }) - expect(snapshot.chunks[1]).toEqual( - expect.objectContaining({ - ref: "grep1:match:1", - kind: "grep_match", - chunkId: "chunk_page_1", - metadata: expect.objectContaining({ - pageNums: [1], - position: 1, - startOffset: 0, - endOffset: 12, - }), - }), - ) + expect(snapshot.chunks.map((chunk) => chunk.ref)).toEqual(["r1:result:1"]) + expect(snapshot.chunks[0]?.content).toBe("Target BP is <130/80 mmHg.") + // chunkType is unknown, but the assetUrl itself has an image extension, + // so asset detection still recognizes it as an image via the URL check. + expect(snapshot.assets).toEqual([ + expect.objectContaining({ ref: "asset:r1:result:1", type: "image" }), + ]) }) - it("copies pageNumbers from the grep match when the SDK provides them", () => { + it("skips agent_explore referencedChunks that lack chunkType instead of throwing", () => { const ledger = createEvidenceLedger() - const snapshot = ledger.addGrepChunksResponse({ - ...makeGrepResponse(), - matches: [ + const snapshot = ledger.addRetrievalResponse({ + namespace: "default", + query: "hypertension CAD blood pressure target", + routerUsed: "agent_explore", + answerText: "", + evidenceText: "[E1] some evidence", + stopReason: "finished", + failureReason: null, + results: [ { - position: 1, - chunkId: "chunk_page_4", - chunkType: "page", - sectionPath: "FINANCIAL SUMMARY", - sourceChunkPath: "pages/page-4.md", - startOffset: 0, - endOffset: 12, - snippet: "automotive revenues", - pageNumbers: [4], + content: "Target BP is <130/80 mmHg.", + chunkType: "text", + score: 0.9, + source: { + documentId: "doc_1", + sourceFileName: "guideline.pdf", + sectionPath: "BP targets", + }, }, ], + // Real agent_explore responses can return referencedChunks entries + // that only carry a summary id, with no chunkType/chunkId/documentId + // even though the SDK type declares those as required strings. + referencedChunks: [ + { summary: "8da0776b-c52b-5602-8579-25c421706f5f" }, + ] as unknown as RetrievalQueryResponse["referencedChunks"], }) - expect(snapshot.chunks[0]).toEqual( - expect.objectContaining({ - ref: "grep1:match:1", - kind: "grep_match", - chunkId: "chunk_page_4", - metadata: expect.objectContaining({ - pageNums: [4], - position: 1, - startOffset: 0, - endOffset: 12, - }), - }), - ) + expect(snapshot.chunks.map((chunk) => chunk.ref)).toEqual(["r1:result:1"]) + expect(snapshot.chunks[0]?.content).toBe("Target BP is <130/80 mmHg.") }) }) @@ -302,77 +260,3 @@ function makePageAssetUrlRetrievalResponse(): RetrievalQueryResponse { ], } } - -function makeReadResponse(): KnowledgeReadResponse { - return { - document: { - localDocumentId: "doc_contract", - documentId: "doc_contract", - jobId: "job_contract", - namespace: "notebook", - sourceFileName: "contract.pdf", - chunkCount: 1, - typeCounts: { text: 0, image: 0, table: 0, page: 1 }, - resultDirectoryPath: "parsed-storage:doc_contract/job_contract", - createdAt: new Date("2026-01-01T00:00:00Z"), - updatedAt: new Date("2026-01-01T00:00:00Z"), - }, - chunks: [ - { - position: 1, - chunkId: "chunk_page_1", - chunkType: "page", - content: "Full page content.", - readableContent: "Full page content.", - sectionPath: "Root / Page 1", - sourceChunkPath: "pages/page-1.md", - filePath: "pages/page-1.png", - assetUrl: "https://assets.example/page-1.png", - pageNumbers: [1], - metadata: { - pageNums: [1], - pageAssets: [ - { - pageNum: 1, - artifactRef: "page_citation_assets/page-1.png", - assetUrl: "https://assets.example/page-1.png", - contentType: "image/png", - }, - ], - }, - }, - ], - } -} - -function makeGrepResponse(): KnowledgeGrepResponse { - return { - document: { - localDocumentId: "doc_contract", - documentId: "doc_contract", - jobId: "job_contract", - namespace: "notebook", - sourceFileName: "contract.pdf", - chunkCount: 4, - typeCounts: { text: 4, image: 0, table: 0, page: 0 }, - resultDirectoryPath: "parsed-storage:doc_contract/job_contract", - createdAt: new Date("2026-01-01T00:00:00Z"), - updatedAt: new Date("2026-01-01T00:00:00Z"), - }, - matches: [ - { - position: 3, - chunkId: "chunk_3", - chunkType: "text", - sectionPath: "Root / Penalties", - sourceChunkPath: "chunks/chunk-3.md", - filePath: "contract.pdf", - startOffset: 10, - endOffset: 17, - snippet: "matched penalty snippet", - }, - ], - scannedChunks: 4, - truncated: false, - } -} diff --git a/src/agent-harness/ledger.ts b/src/agent-harness/ledger.ts index 42f31207..9c80448d 100644 --- a/src/agent-harness/ledger.ts +++ b/src/agent-harness/ledger.ts @@ -1,8 +1,4 @@ import type { - KnowledgeGrepMatch, - KnowledgeGrepResponse, - KnowledgeReadChunk, - KnowledgeReadResponse, RetrievalQueryResponse, RetrievalResult, } from "@ontos-ai/knowhere-sdk" @@ -18,8 +14,6 @@ const imageExtensions = [".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg"] as co type MutableLedger = { retrievalCount: number - readCount: number - grepCount: number chunks: EvidenceChunk[] assets: EvidenceAsset[] evidenceText: string[] @@ -47,8 +41,6 @@ export type EvidenceLedger = ReturnType export function createEvidenceLedger() { const ledger: MutableLedger = { retrievalCount: 0, - readCount: 0, - grepCount: 0, chunks: [], assets: [], evidenceText: [], @@ -84,6 +76,15 @@ export function createEvidenceLedger() { }) response.referencedChunks.forEach((chunk, index) => { + // Knowhere's agent_explore router returns referencedChunks entries + // that may carry only a summary id with no chunkType/content + // (despite the SDK type declaring chunkType as required). Skip + // entries missing a usable chunkType: they have no real content + // (content is always "" here) and chunkType is required downstream + // (asset-type detection calls chunkType.toLowerCase()). + if (typeof chunk.chunkType !== "string" || chunk.chunkType.trim().length === 0) { + return + } const content = "" addChunk({ ledger, @@ -112,38 +113,6 @@ export function createEvidenceLedger() { return snapshot(ledger) }, - addReadChunksResponse(response: KnowledgeReadResponse): EvidenceLedgerSnapshot { - ledger.readCount += 1 - const readIndex = ledger.readCount - - response.chunks.forEach((chunk, index) => { - addChunkFromReadChunk({ - ledger, - response, - chunk, - ref: `read${readIndex}:chunk:${index + 1}`, - }) - }) - - return snapshot(ledger) - }, - - addGrepChunksResponse(response: KnowledgeGrepResponse): EvidenceLedgerSnapshot { - ledger.grepCount += 1 - const grepIndex = ledger.grepCount - - response.matches.forEach((match, index) => { - addChunkFromGrepMatch({ - ledger, - response, - match, - ref: `grep${grepIndex}:match:${index + 1}`, - }) - }) - - return snapshot(ledger) - }, - read(ref: string, offset = 0, limit = 4_000) { const chunk = ledger.chunks.find((candidate) => candidate.ref === ref) if (!chunk) { @@ -177,13 +146,6 @@ export function createEvidenceLedger() { return ledger.chunks.length > 0 || ledger.evidenceText.length > 0 }, - hasRef(ref: string): boolean { - return ( - ledger.chunks.some((chunk) => chunk.ref === ref) || - ledger.assets.some((asset) => asset.ref === ref) - ) - }, - snapshot(): EvidenceLedgerSnapshot { return snapshot(ledger) }, @@ -219,100 +181,6 @@ function addChunkFromResult(input: { }) } -function addChunkFromReadChunk(input: { - readonly ledger: MutableLedger - readonly response: KnowledgeReadResponse - readonly chunk: KnowledgeReadChunk - readonly ref: string -}): void { - addChunk({ - ledger: input.ledger, - chunk: { - ref: input.ref, - kind: "read_chunk", - chunkId: input.chunk.chunkId, - content: input.chunk.content, - contentPreview: buildContentPreview(input.chunk.content), - chunkType: input.chunk.chunkType, - score: null, - sourceChunkPath: input.chunk.sourceChunkPath, - filePath: input.chunk.filePath, - metadata: input.chunk.metadata, - source: { - documentId: input.response.document.documentId, - sourceFileName: input.response.document.sourceFileName, - sectionPath: input.chunk.sectionPath, - }, - revisionKey: input.response.document.jobId, - ...(input.chunk.assetUrl ? { assetUrl: input.chunk.assetUrl } : {}), - }, - }) -} - -function addChunkFromGrepMatch(input: { - readonly ledger: MutableLedger - readonly response: KnowledgeGrepResponse - readonly match: KnowledgeGrepMatch - readonly ref: string -}): void { - const donor = input.ledger.chunks.find( - (chunk) => - chunk.chunkId === input.match.chunkId && hasPageMetadata(chunk.metadata), - ) - const pageNums = - input.match.pageNumbers && input.match.pageNumbers.length > 0 - ? [...input.match.pageNumbers] - : undefined - - addChunk({ - ledger: input.ledger, - chunk: { - ref: input.ref, - kind: "grep_match", - chunkId: input.match.chunkId, - content: input.match.snippet, - contentPreview: buildContentPreview(input.match.snippet), - chunkType: input.match.chunkType, - score: null, - sourceChunkPath: input.match.sourceChunkPath, - filePath: input.match.filePath, - metadata: { - ...(donor?.metadata ?? {}), - ...(pageNums ? { pageNums } : {}), - position: input.match.position, - startOffset: input.match.startOffset, - endOffset: input.match.endOffset, - }, - source: { - documentId: input.response.document.documentId, - sourceFileName: input.response.document.sourceFileName, - sectionPath: input.match.sectionPath, - }, - revisionKey: input.response.document.jobId, - }, - }) -} - -function hasPageMetadata( - metadata: Readonly> | undefined, -): boolean { - if (!metadata) return false - const values = [ - metadata.pageNums, - metadata.page_nums, - metadata.pageNum, - metadata.page_num, - metadata.pageAssets, - metadata.page_assets, - ] - return values.some((value) => { - if (Array.isArray(value)) return value.length > 0 - if (typeof value === "number") return Number.isSafeInteger(value) && value > 0 - if (typeof value === "string") return value.trim().length > 0 - return false - }) -} - function addChunk(input: { readonly ledger: MutableLedger readonly chunk: Omit @@ -347,8 +215,16 @@ function buildContentPreview(content: string): string { return `${normalized.slice(0, contentPreviewLimit)}...` } +// Knowhere's chunkType is declared as a required string in the SDK type, +// but real API responses (seen on referencedChunks; results are the same +// contract) can omit it. Normalize defensively instead of calling +// .toLowerCase() on a value that may be undefined at runtime. +function normalizeChunkType(chunkType: string): string { + return typeof chunkType === "string" ? chunkType.toLowerCase() : "" +} + function isRenderableAsset(chunkType: string, assetUrl: string): boolean { - const normalizedChunkType = chunkType.toLowerCase() + const normalizedChunkType = normalizeChunkType(chunkType) return ( normalizedChunkType === "image" || normalizedChunkType === "table" || @@ -379,7 +255,7 @@ function getEvidenceAssetCandidate( function getPageCitationAssetCandidate( chunk: Omit, ): EvidenceAssetCandidate | null { - if (chunk.chunkType.toLowerCase() !== "page") return null + if (normalizeChunkType(chunk.chunkType) !== "page") return null const candidates = [ ...parsePageCitationAssetCandidates(chunk.metadata?.pageAssets), @@ -409,7 +285,7 @@ function getPageCitationAssetCandidate( } function getAssetType(chunkType: string, assetUrl: string): "image" | "table" { - return chunkType.toLowerCase() === "table" && !isImageAssetUrl(assetUrl) + return normalizeChunkType(chunkType) === "table" && !isImageAssetUrl(assetUrl) ? "table" : "image" } diff --git a/src/agent-harness/runtime.test.ts b/src/agent-harness/runtime.test.ts index b459d5b8..11ac3d94 100644 --- a/src/agent-harness/runtime.test.ts +++ b/src/agent-harness/runtime.test.ts @@ -28,6 +28,14 @@ describe("agent harness runtime", () => { expect(prompt).toContain( "Call knowhere_search only when memory is insufficient and groundingPolicy requires citing source documents", ) + expect(prompt).toContain( + "call knowhere_search again with a refined query", + ) + expect(prompt).toContain("Refine knowhere_search at most twice") + expect(prompt).not.toContain("knowhere_list_documents") + expect(prompt).not.toContain("knowhere_get_document_outline") + expect(prompt).not.toContain("knowhere_read_chunks") + expect(prompt).not.toContain("knowhere_grep_chunks") expect(prompt).toContain( "Do not treat every question as a document-retrieval task", ) @@ -42,16 +50,14 @@ describe("agent harness runtime", () => { expect(prompt).not.toContain("navigation action") }) - it("tells the agent citation metadata is optional and ledger-resolved", () => { + it("tells the agent to cite by search-chunk pick numbers", () => { const prompt = buildHarnessSystemPrompt(makeTurnInput()) - expect(prompt).toContain( - "Citation label and source metadata are optional", - ) - expect(prompt).toContain( - "Notebook resolves citation metadata from evidence refs when possible", - ) - expect(prompt).not.toContain("must match the selected evidence ref exactly") + expect(prompt).toContain("citations is a list of { pick }") + expect(prompt).toContain("Notebook writes the citation list from those picks") + expect(prompt).toContain("Do not pass documentId or evidence refs as citations") + expect(prompt).not.toContain("Citation refs must be evidence refs") + expect(prompt).not.toContain("Citation label and source metadata are optional") }) it("tells the agent to emit [[cite:n]] markers instead of title/pN or [1]", () => { @@ -98,6 +104,7 @@ describe("agent harness runtime", () => { }) expect(result).toContain('') + expect(result).toContain('pick="1"') expect(result).toContain('ref="r1:result:1"') expect(result).toContain('ref="asset:r1:result:1"') expect(query).toHaveBeenCalledWith({ @@ -169,10 +176,12 @@ describe("agent harness runtime", () => { const firstResult = await executeTool(tools.knowhere_search, { query: "first" }) const secondResult = await executeTool(tools.knowhere_search, { query: "second" }) + expect(firstResult).toContain('pick="1"') expect(firstResult).toContain('ref="r1:result:1"') expect(secondResult).toContain('retrievalCount="2"') expect(secondResult).toContain("Second retrieval evidence.") expect(secondResult).not.toContain("") + expect(secondResult).toContain('pick="2"') expect(secondResult).toContain('ref="r2:result:1"') expect(JSON.stringify(secondResult)).not.toContain("r1:result:1") expect(ledger.snapshot().chunks.map((chunk) => chunk.ref)).toEqual([ @@ -471,7 +480,7 @@ describe("agent harness runtime", () => { expect( await executeTool(tools.finalize, { text: "Revenue was $24.9B [[cite:1]] [[cite:2]].", - citations: [{ ref: "r1:result:1" }, { ref: "r1:result:2" }], + citations: [{ pick: 1 }, { pick: 2 }], memoryCitations: [], artifacts: [], unresolved: [], @@ -598,7 +607,7 @@ describe("agent harness runtime", () => { const finalize = await executeTool(tools.finalize, { text: "The amount is 5000 yuan [[cite:1]].", - citations: [{ ref: "r1:referenced:1" }], + citations: [{ pick: 1 }], memoryCitations: [], artifacts: [], unresolved: [], @@ -610,6 +619,98 @@ describe("agent harness runtime", () => { expect(state.finalized).not.toBe(true) }) + it("writes citation refs from ledger picks and rejects picks outside the ledger", async () => { + const ledger = createEvidenceLedger() + ledger.addRetrievalResponse(makeRetrievalResponse()) + const state: { + finalized?: boolean + finalizedManifest?: OutputManifest + } = {} + const tools = createHarnessTools({ + state, + ledger, + memoryTools: makeMemoryTools(), + knowhereTools: makeKnowhereTools(), + recentTurns: [], + }) + + const rejected = await executeTool(tools.finalize, { + text: "Target is <130/80 mmHg [[cite:1]].", + citations: [{ pick: 99 }], + memoryCitations: [], + artifacts: [], + unresolved: [], + }) + + expect(rejected).toMatchObject({ + ok: false, + unknownPicks: [99], + }) + expect(String((rejected as { message: string }).message)).toContain( + "Available picks: 1-1", + ) + expect(state.finalized).not.toBe(true) + + const accepted = await executeTool(tools.finalize, { + text: "Target is <130/80 mmHg [[cite:1]].", + citations: [{ pick: 1 }], + memoryCitations: [], + artifacts: [], + unresolved: [], + }) + expect(accepted).toMatchObject({ + ok: true, + citations: [{ ref: "r1:result:1" }], + }) + expect(state.finalized).toBe(true) + expect(state.finalizedManifest?.citations).toEqual([{ ref: "r1:result:1" }]) + }) + + it("maps finalize picks across successive searches", async () => { + const ledger = createEvidenceLedger() + ledger.addRetrievalResponse(makeRetrievalResponse()) + ledger.addRetrievalResponse({ + ...makeRetrievalResponse(), + query: "second query", + results: [ + { + content: "Second retrieval evidence.", + chunkType: "text", + score: 0.8, + source: { + documentId: "doc_2", + sourceFileName: "second.pdf", + sectionPath: "Second", + }, + }, + ], + }) + const state: { + finalizedManifest?: OutputManifest + } = {} + const tools = createHarnessTools({ + state, + ledger, + memoryTools: makeMemoryTools(), + knowhereTools: makeKnowhereTools(), + recentTurns: [], + }) + + const accepted = await executeTool(tools.finalize, { + text: "Second source [[cite:1]].", + citations: [{ pick: 2 }], + memoryCitations: [], + artifacts: [], + unresolved: [], + }) + + expect(accepted).toMatchObject({ + ok: true, + citations: [{ ref: "r2:result:1" }], + }) + expect(state.finalizedManifest?.citations).toEqual([{ ref: "r2:result:1" }]) + }) + it("accepts finalize output without planning-tool gating", async () => { const state: { finalizedManifest?: OutputManifest @@ -719,7 +820,7 @@ describe("agent harness runtime", () => { }) const manifest = { text: "The contractor pays 5000 yuan per occurrence [[cite:1]].", - citations: [{ ref: "r1:referenced:1" }], + citations: [{ pick: 1 }], memoryCitations: [], artifacts: [], unresolved: [], @@ -781,7 +882,7 @@ describe("agent harness runtime", () => { const result = await executeTool(tools.finalize, { text: "The contractor pays 5000 yuan [[cite:1]].", - citations: [{ ref: "r1:result:1" }], + citations: [{ pick: 1 }], memoryCitations: [], artifacts: [], unresolved: [], @@ -1024,6 +1125,15 @@ describe("agent harness runtime", () => { expect(result.activeTools).not.toContain("knowhere_search") }) + it("forces a tool call on ordinary steps so the model cannot skip finalize with bare text", () => { + const result = prepareHarnessStep({ + stepNumber: 3, + messages: [], + }) + + expect(result.toolChoice).toBe("required") + }) + it("opens memory_search and knowhere_search together as peers when sources are required", () => { const result = prepareHarnessStep({ stepNumber: 3, @@ -1039,15 +1149,12 @@ describe("agent harness runtime", () => { }) expect(result.activeTools).toEqual( - expect.arrayContaining([ - "memory_search", - "knowhere_search", - "knowhere_list_documents", - "knowhere_get_document_outline", - "knowhere_read_chunks", - "knowhere_grep_chunks", - ]), + expect.arrayContaining(["memory_search", "knowhere_search"]), ) + expect(result.activeTools).not.toContain("knowhere_list_documents") + expect(result.activeTools).not.toContain("knowhere_get_document_outline") + expect(result.activeTools).not.toContain("knowhere_read_chunks") + expect(result.activeTools).not.toContain("knowhere_grep_chunks") }) it("forces image inspection before forced finalization when image assets are available", () => { @@ -1195,10 +1302,6 @@ function makeKnowhereTools( ): KnowhereToolRuntime { return { search, - listDocuments: vi.fn().mockResolvedValue({ documents: [] }), - getDocumentOutline: vi.fn().mockRejectedValue(new Error("Not configured.")), - readChunks: vi.fn().mockRejectedValue(new Error("Not configured.")), - grepChunks: vi.fn().mockRejectedValue(new Error("Not configured.")), } } diff --git a/src/agent-harness/runtime.ts b/src/agent-harness/runtime.ts index 0f4aab0a..43d9329a 100644 --- a/src/agent-harness/runtime.ts +++ b/src/agent-harness/runtime.ts @@ -18,6 +18,7 @@ import type { ContextPolicy, EvidenceAsset, EvidenceChunk, + EvidenceLedgerSnapshot, HarnessRunResult, HarnessToolCallTrace, HarnessTrace, @@ -30,6 +31,8 @@ import type { KnowhereToolRuntime, MemorySearchKind, MemoryToolRuntime, + OutputArtifactView, + OutputCitation, OutputManifest, } from "./types" import { memorySearchKinds } from "./types" @@ -67,10 +70,12 @@ type HarnessTools = ReturnType type HarnessStepPreparation = { messages: ModelMessage[] activeTools?: Array> - toolChoice?: { - type: "tool" - toolName: Extract - } + toolChoice?: + | "required" + | { + type: "tool" + toolName: Extract + } } const targetModalitySchema = z.enum(["text", "image", "table"]) @@ -82,14 +87,6 @@ const knowhereSearchTargetContentSchema = z.enum([ "text_image", "text_table", ]) -const knowledgeChunkTypeSchema = z.enum(["text", "image", "table", "page"]) - -const knowhereDocumentReferenceSchema = z.object({ - localDocumentId: z.string().min(1).optional(), - documentId: z.string().min(1).optional(), - jobId: z.string().min(1).optional(), - revisionKey: z.string().min(1).optional(), -}) const knowhereSearchSchema = z.object({ query: z.string().min(1), @@ -101,27 +98,6 @@ const knowhereSearchSchema = z.object({ threshold: z.number().min(0).max(1).optional(), }) -const knowhereReadChunksSchema = knowhereDocumentReferenceSchema.extend({ - page: z.number().int().min(1).optional(), - pageSize: z.number().int().min(1).max(50).optional(), - sectionPath: z.string().min(1).optional(), - startChunk: z.number().int().min(0).optional(), - endChunk: z.number().int().min(0).optional(), - chunkId: z.string().min(1).optional(), - chunkType: knowledgeChunkTypeSchema.optional(), -}) - -const knowhereGrepChunksSchema = knowhereDocumentReferenceSchema.extend({ - pattern: z.string().min(1), - continuationCursor: z.string().min(1).optional(), - isRegex: z.boolean().optional(), - isCaseSensitive: z.boolean().optional(), - maxResults: z.number().int().min(1).max(50).optional(), - chunkType: knowledgeChunkTypeSchema.optional(), - sectionPathPrefix: z.string().min(1).optional(), - contextChars: z.number().int().min(0).max(2_000).optional(), -}) - const intentFrameSchema = z.object({ task: z.enum([ "answer", @@ -164,16 +140,8 @@ const contextPolicySchema = z.object({ activePriorTurnIds: z.array(z.string()).default([]), }) -const outputCitationSchema = z.object({ - ref: z.string().min(1), - label: z.string().min(1).optional(), - source: z - .object({ - documentId: z.string().nullable().optional(), - sourceFileName: z.string().nullable().optional(), - sectionPath: z.string().nullable().optional(), - }) - .optional(), +const citationPickSchema = z.object({ + pick: z.number().int().positive(), }) const memoryCitationSchema = z.object({ @@ -210,9 +178,9 @@ const outputArtifactSchema = z.union([ derivedTableArtifactSchema, ]) -const outputManifestSchema = z.object({ +const finalizeManifestSchema = z.object({ text: z.string(), - citations: z.array(outputCitationSchema).default([]), + citations: z.array(citationPickSchema).default([]), memoryCitations: z.array(memoryCitationSchema).default([]), artifacts: z.array(outputArtifactSchema).default([]), unresolved: z.array(z.string()).default([]), @@ -286,13 +254,7 @@ const alwaysAvailableTools = [ const fluidRetrievalTools = ["memory_search"] as const -const crystalRetrievalTools = [ - "knowhere_search", - "knowhere_list_documents", - "knowhere_get_document_outline", - "knowhere_read_chunks", - "knowhere_grep_chunks", -] as const +const crystalRetrievalTools = ["knowhere_search"] as const /** Reserved third retrieval slot (cognition). Not registered this round. */ const cognitionRetrievalTools = [] as const @@ -358,6 +320,10 @@ export function prepareHarnessStep(input: { activeTools: selectHarnessActiveTools({ intent: input.intent, }), + // finalize is the only output contract (see its tool description). Force + // a tool call every step so the model cannot end the turn with a bare + // text response that skips finalize's citation/artifact validation. + toolChoice: "required", } } @@ -448,7 +414,7 @@ type ModelMessageForRole = Extract< function buildForcedFinalizationFeedback(): string { return [ "The retrieval step budget has been reached.", - "Do not search again or call any Knowhere evidence-reading tools.", + "Do not search again.", "Use only the evidence and tool results already available in this turn.", "Call finalize now with the best supported answer.", "If the existing evidence is insufficient, explain the gap in unresolved", @@ -551,7 +517,7 @@ export function createHarnessTools(input: { knowhere_search: tool({ description: - "Search Knowhere for relevant Notebook evidence. Returns tagged text with evidence refs such as r1:result:1 and asset refs such as asset:r1:result:1.", + "Search Knowhere for relevant Notebook evidence. Returns tagged text with pick numbers for finalize citations, evidence refs such as r1:result:1, and asset refs such as asset:r1:result:1.", inputSchema: knowhereSearchSchema, execute: async (request) => traceToolCall(input.state, { @@ -567,83 +533,6 @@ export function createHarnessTools(input: { }), }), - knowhere_list_documents: tool({ - description: - "List ready visible Notebook/Knowhere documents available for this chat turn. Use this to discover documentId and revisionKey before outline/read/grep.", - inputSchema: z.object({}), - execute: async () => - traceToolCall(input.state, { - toolName: "knowhere_list_documents", - inputSummary: {}, - execute: async () => - executeKnowhereTextTool({ - operation: "list_documents", - execute: async () => - knowhereToolText.formatListDocuments( - await input.knowhereTools.listDocuments(), - ), - }), - summarizeOutput: summarizeKnowhereTextOutput, - }), - }), - - knowhere_get_document_outline: tool({ - description: - "Read a document outline from Knowhere parsed storage. Use documentId/revisionKey from knowhere_list_documents or search refs.", - inputSchema: knowhereDocumentReferenceSchema, - execute: async (request) => - traceToolCall(input.state, { - toolName: "knowhere_get_document_outline", - inputSummary: summarizeDocumentReference(request), - execute: async () => - executeKnowhereTextTool({ - operation: "get_document_outline", - validate: () => validateDocumentReference(request), - execute: async () => - knowhereToolText.formatOutline( - await input.knowhereTools.getDocumentOutline(request), - ), - }), - summarizeOutput: summarizeKnowhereTextOutput, - }), - }), - - knowhere_read_chunks: tool({ - description: - "Read complete chunk bodies from Knowhere parsed storage. This tool never slices individual chunk content; control read size with page/pageSize, sectionPath, startChunk/endChunk, chunkId, and chunkType.", - inputSchema: knowhereReadChunksSchema, - execute: async (request) => - traceToolCall(input.state, { - toolName: "knowhere_read_chunks", - inputSummary: summarizeReadChunksRequest(request), - execute: async () => - executeKnowhereReadChunks({ - ledger: input.ledger, - knowhereTools: input.knowhereTools, - request, - }), - summarizeOutput: summarizeKnowhereTextOutput, - }), - }), - - knowhere_grep_chunks: tool({ - description: - "Search chunk text with a literal or regex pattern. Returns bounded match snippets as grep refs such as grep1:match:1 and may include truncated=true with a continuationCursor.", - inputSchema: knowhereGrepChunksSchema, - execute: async (request) => - traceToolCall(input.state, { - toolName: "knowhere_grep_chunks", - inputSummary: summarizeGrepChunksRequest(request), - execute: async () => - executeKnowhereGrepChunks({ - ledger: input.ledger, - knowhereTools: input.knowhereTools, - request, - }), - summarizeOutput: summarizeKnowhereTextOutput, - }), - }), - inspectImage: tool({ description: "Inspect cited Knowhere page/image asset refs for OCR, visual details, and provenance boxes. Call this after retrieval and before finalize whenever the answer cites page or image assets.", @@ -730,17 +619,42 @@ export function createHarnessTools(input: { description: "Finalize the user-facing output manifest. This is the only final answer " + "contract. Artifacts listed here with display=true are the exact set of " + - "images/tables shown to the user; cite evidence refs when available. " + - "Use citations for Knowhere evidence and memoryCitations for fluid memory refs. " + + "images/tables shown to the user. citations is the list of evidence picks " + + "you used; each pick is the pick number on a Knowhere search chunk. " + + "Notebook writes citation refs from the evidence ledger. " + + "Use memoryCitations for fluid memory refs. " + "Cited page/image assets must be inspected with inspectImage first.", - inputSchema: outputManifestSchema, + inputSchema: finalizeManifestSchema, execute: async (manifest) => traceToolCall(input.state, { toolName: "finalize", inputSummary: summarizeManifest(manifest), execute: async () => { + const resolvedCitations = resolveCitationPicks({ + citations: manifest.citations, + ledger: input.ledger, + }) + if (!resolvedCitations.ok) { + return { + ok: false as const, + message: buildFinalizeRequiresPicksMessage({ + unknownPicks: resolvedCitations.unknownPicks, + ledger: input.ledger.snapshot(), + }), + unknownPicks: resolvedCitations.unknownPicks, + } + } + + const outputManifest: OutputManifest = { + text: manifest.text, + citations: resolvedCitations.citations, + memoryCitations: manifest.memoryCitations, + artifacts: manifest.artifacts, + unresolved: manifest.unresolved, + } + const inspectRefs = getUninspectedCitedImageRefs({ - manifest, + manifest: outputManifest, ledger: input.ledger, inspectedImageRefs: input.state.inspectedImageRefs ?? [], inspectImagesAvailable: input.inspectImages !== undefined, @@ -753,9 +667,9 @@ export function createHarnessTools(input: { } } - input.state.finalizedManifest = manifest + input.state.finalizedManifest = outputManifest input.state.finalized = true - return { ok: true as const, ...manifest } + return { ok: true as const, ...outputManifest } }, summarizeOutput: summarizeFinalizeOutput, }), @@ -924,6 +838,50 @@ async function inspectRetrievedImages(input: { } } +function resolveCitationPicks(input: { + readonly citations: readonly { pick: number }[] + readonly ledger: ReturnType +}): + | { ok: true; citations: OutputCitation[] } + | { ok: false; unknownPicks: number[] } { + const chunks = input.ledger.snapshot().chunks + const unknownPicks: number[] = [] + const citations: OutputCitation[] = [] + + for (const citation of input.citations) { + const chunk = Number.isInteger(citation.pick) + ? chunks[citation.pick - 1] + : undefined + if (!chunk) { + if (!unknownPicks.includes(citation.pick)) { + unknownPicks.push(citation.pick) + } + continue + } + citations.push({ ref: chunk.ref }) + } + + if (unknownPicks.length > 0) { + return { ok: false, unknownPicks } + } + return { ok: true, citations } +} + +function buildFinalizeRequiresPicksMessage(input: { + readonly unknownPicks: readonly number[] + readonly ledger: EvidenceLedgerSnapshot +}): string { + const availableCount = input.ledger.chunks.length + return [ + "Citations must use pick numbers from Knowhere search chunks.", + `Unknown citation picks: ${input.unknownPicks.join(" ")}.`, + availableCount > 0 + ? `Available picks: 1-${availableCount}.` + : "No evidence picks are available; list the gap in unresolved and omit citations.", + "Call finalize again using only available picks.", + ].join(" ") +} + function getUninspectedCitedImageRefs(input: { readonly manifest: OutputManifest readonly ledger: ReturnType @@ -1009,26 +967,10 @@ function buildFinalizeRequiresInspectionMessage( ].join(" ") } -type KnowhereToolOperation = - | "search" - | "list_documents" - | "get_document_outline" - | "read_chunks" - | "grep_chunks" +type KnowhereToolOperation = "search" type MemorySearchToolRequest = z.infer type KnowhereSearchToolRequest = z.infer -type KnowhereDocumentReferenceRequest = z.infer< - typeof knowhereDocumentReferenceSchema -> -type KnowhereReadChunksToolRequest = z.infer -type KnowhereGrepChunksToolRequest = z.infer -type DocumentReferenceSummary = { - readonly documentId?: string - readonly localDocumentId?: string - readonly hasJobId: boolean - readonly hasRevisionKey: boolean -} async function executeMemorySearch(input: { readonly memoryTools: MemoryToolRuntime @@ -1070,48 +1012,7 @@ async function executeKnowhereSearch(input: { return knowhereToolText.formatSearch({ response, retrievalCount: snapshot.retrievalCount, - chunks: snapshot.chunks.slice(beforeSnapshot.chunks.length), - assets: snapshot.assets.slice(beforeSnapshot.assets.length), - }) - }, - }) -} - -async function executeKnowhereReadChunks(input: { - readonly ledger: ReturnType - readonly knowhereTools: KnowhereToolRuntime - readonly request: KnowhereReadChunksToolRequest -}): Promise { - return executeKnowhereTextTool({ - operation: "read_chunks", - validate: () => validateDocumentReference(input.request), - execute: async () => { - const beforeSnapshot = input.ledger.snapshot() - const response = await input.knowhereTools.readChunks(input.request) - const snapshot = input.ledger.addReadChunksResponse(response) - return knowhereToolText.formatReadChunks({ - response, - chunks: snapshot.chunks.slice(beforeSnapshot.chunks.length), - assets: snapshot.assets.slice(beforeSnapshot.assets.length), - }) - }, - }) -} - -async function executeKnowhereGrepChunks(input: { - readonly ledger: ReturnType - readonly knowhereTools: KnowhereToolRuntime - readonly request: KnowhereGrepChunksToolRequest -}): Promise { - return executeKnowhereTextTool({ - operation: "grep_chunks", - validate: () => validateDocumentReference(input.request), - execute: async () => { - const beforeSnapshot = input.ledger.snapshot() - const response = await input.knowhereTools.grepChunks(input.request) - const snapshot = input.ledger.addGrepChunksResponse(response) - return knowhereToolText.formatGrepChunks({ - response, + chunkPickStart: beforeSnapshot.chunks.length, chunks: snapshot.chunks.slice(beforeSnapshot.chunks.length), assets: snapshot.assets.slice(beforeSnapshot.assets.length), }) @@ -1121,17 +1022,8 @@ async function executeKnowhereGrepChunks(input: { async function executeKnowhereTextTool(input: { readonly operation: KnowhereToolOperation - readonly validate?: () => string | null readonly execute: () => Promise }): Promise { - const validationError = input.validate?.() - if (validationError) { - return knowhereToolText.formatError({ - operation: input.operation, - message: validationError, - }) - } - try { return await input.execute() } catch (error) { @@ -1142,20 +1034,6 @@ async function executeKnowhereTextTool(input: { } } -function validateDocumentReference( - request: KnowhereDocumentReferenceRequest, -): string | null { - if ( - request.documentId || - request.localDocumentId || - request.jobId - ) { - return null - } - - return "A documentId, localDocumentId, or jobId is required." -} - function getUniqueTrimmedRefs(refs: readonly string[]): string[] { const normalizedRefs: string[] = [] for (const ref of refs) { @@ -1276,48 +1154,6 @@ function summarizeKnowhereSearchRequest(request: { } } -function summarizeDocumentReference( - request: KnowhereDocumentReferenceRequest, -): DocumentReferenceSummary { - return { - documentId: request.documentId, - localDocumentId: request.localDocumentId, - hasJobId: typeof request.jobId === "string", - hasRevisionKey: typeof request.revisionKey === "string", - } -} - -function summarizeReadChunksRequest( - request: KnowhereReadChunksToolRequest, -): unknown { - return { - ...summarizeDocumentReference(request), - page: request.page, - pageSize: request.pageSize, - sectionPath: request.sectionPath, - startChunk: request.startChunk, - endChunk: request.endChunk, - chunkId: request.chunkId, - chunkType: request.chunkType, - } -} - -function summarizeGrepChunksRequest( - request: KnowhereGrepChunksToolRequest, -): unknown { - return { - ...summarizeDocumentReference(request), - patternLength: request.pattern.trim().length, - continuationCursor: request.continuationCursor, - isRegex: request.isRegex, - isCaseSensitive: request.isCaseSensitive, - maxResults: request.maxResults, - chunkType: request.chunkType, - sectionPathPrefix: request.sectionPathPrefix, - contextChars: request.contextChars, - } -} - function summarizeKnowhereTextOutput(output: unknown): unknown { if (typeof output !== "string") return output return { @@ -1377,7 +1213,13 @@ function summarizeReadPriorTurnOutput(output: unknown): unknown { } } -function summarizeManifest(manifest: OutputManifest): unknown { +function summarizeManifest(manifest: { + readonly text: string + readonly citations: readonly unknown[] + readonly memoryCitations: readonly unknown[] + readonly artifacts: readonly OutputArtifactView[] + readonly unresolved: readonly string[] +}): unknown { return { textLength: manifest.text.length, citationCount: manifest.citations.length, @@ -1428,15 +1270,15 @@ export function buildHarnessSystemPrompt(turn: AgentTurnInput): string { "1. Call declareIntent when it helps you plan the response. Capture constraints like a requested image/table count in constraints.desiredCount.", "2. Call setContextPolicy when prior turns may influence this turn.", "3. When the policy needs prior-turn detail (references or corrections), call readPriorTurn for the relevant ids.", - "4. Call memory_search first when known fluid memory may answer the request. Call knowhere_search only when memory is insufficient and groundingPolicy requires citing source documents. Use knowhere_list_documents, knowhere_get_document_outline, knowhere_read_chunks, and knowhere_grep_chunks for focused document reads.", + "4. Call memory_search first when known fluid memory may answer the request. Call knowhere_search when memory is insufficient and groundingPolicy requires citing source documents. If the returned evidence is not enough to answer, or the query needs to focus differently, call knowhere_search again with a refined query (different keywords / topK / targetContent) instead of trying to browse documents directly. Knowhere's own retrieval agent already navigates the corpus internally.", "5. After Knowhere returns image/page asset refs, call inspectImage on the page/image assets you will cite before finalize. This supplies OCR/visual context and provenance boxes.", "6. Inspect each unique cited page once; retrieval already bounds the available evidence set.", - "7. knowhere_read_chunks returns complete chunk bodies; control size with page/pageSize, sectionPath, startChunk/endChunk, chunkId, and chunkType.", - "8. Call finalize with text, citations, artifacts, and unresolved issues when you are ready to answer.", + "7. Call finalize with text, citations, artifacts, and unresolved issues when you are ready to answer.", "", "Retrieval rules:", "- First use memory_search to see whether known fluid memory can answer directly.", "- Call knowhere_search only when memory is insufficient and groundingPolicy requires citing source documents.", + "- Refine knowhere_search at most twice. If two refined searches still do not add new relevant evidence, call finalize and list the gap in unresolved.", "- Do not treat every question as a document-retrieval task.", "", "Context rules:", @@ -1448,13 +1290,12 @@ export function buildHarnessSystemPrompt(turn: AgentTurnInput): string { "- Final output is the OutputManifest passed to finalize, not freeform tool JSON or trailing text.", "- artifacts with display=true are the exact images/tables shown. Never display every candidate; honor constraints.desiredCount / maxCount.", "- Use type=derived_table only for tables you create from evidence; every derived_table.sourceRefs entry must reference evidence in the ledger.", - "- Prefer citation and selected image/table artifact refs returned by Knowhere tools in the evidence ledger.", + "- citations is a list of { pick }. pick is the 1-based number on the search chunk you are using. Notebook writes the citation list from those picks. Do not pass documentId or evidence refs as citations.", "- Place [[cite:n]] immediately after the supported claim. n is the 1-based index into the citations array passed to finalize.", "- Write one marker per index: [[cite:1]] [[cite:3]] [[cite:5]]. Never group indices as [[cite:1, 3, 5]].", "- Do not write title/pN, [1], Markdown footnotes, or [Source N: ...] in the answer text. Notebook renders chips from [[cite:n]] and citation metadata.", "- Repeat [[cite:n]] when another claim uses the same page. Do not collapse same-page citations to one row.", - "- Citation label and source metadata are optional. Notebook resolves citation metadata from evidence refs when possible.", - "- If evidence is relevant but you cannot identify a supporting evidence ref, answer with unresolved issues instead of fabricating a ref.", + "- If you have no supporting evidence pick, omit citations and list the gap in unresolved.", "- inspectImage observations are inspection notes, not new source refs. Final citations and displayed image artifacts must use the original retrieved image asset refs.", "- Do not finalize cited page/image assets from chunk text alone when inspectImage is available. Inspect those asset refs first, then write the answer using the inspection notes.", "- If text evidence identifies a relevant page/image but does not include the exact fact, inspect the returned image asset for OCR/detail before saying the answer is unavailable.", diff --git a/src/agent-harness/types.ts b/src/agent-harness/types.ts index 82f38ab5..ad686ad6 100644 --- a/src/agent-harness/types.ts +++ b/src/agent-harness/types.ts @@ -1,10 +1,4 @@ import type { - KnowledgeDocumentReference, - KnowledgeGrepParams, - KnowledgeGrepResponse, - KnowledgeOutline, - KnowledgeReadParams, - KnowledgeReadResponse, RetrievalQueryParams, RetrievalQueryResponse, } from "@ontos-ai/knowhere-sdk" @@ -99,36 +93,10 @@ export type KnowhereSearchRequest = Pick< readonly purpose?: string } -export type KnowhereDocumentSummary = { - readonly documentId?: string - readonly localDocumentId?: string - readonly revisionKey?: string - readonly namespace?: string - readonly sourceFileName: string - readonly title?: string - readonly status?: string - readonly chunkCount?: number - readonly typeCounts?: Readonly> -} - -export type KnowhereListDocumentsResponse = { - readonly documents: readonly KnowhereDocumentSummary[] -} - export type KnowhereToolRuntime = { readonly search: ( input: KnowhereSearchRequest, ) => Promise - readonly listDocuments: () => Promise - readonly getDocumentOutline: ( - input: KnowledgeDocumentReference, - ) => Promise - readonly readChunks: ( - input: KnowledgeReadParams, - ) => Promise - readonly grepChunks: ( - input: KnowledgeGrepParams, - ) => Promise } export const memorySearchKinds = [ @@ -170,7 +138,7 @@ export type MemoryCitation = { export type EvidenceChunk = { readonly ref: string - readonly kind: "result" | "referenced_chunk" | "read_chunk" | "grep_match" + readonly kind: "result" | "referenced_chunk" readonly chunkId?: string readonly content: string readonly contentPreview: string diff --git a/src/components/chat-message-list.tsx b/src/components/chat-message-list.tsx index 796f2eab..2b7a5d13 100644 --- a/src/components/chat-message-list.tsx +++ b/src/components/chat-message-list.tsx @@ -931,10 +931,14 @@ function isImageCitation( citation: ChatCitationView, assetUrl: string, ): boolean { - return ( - citation.chunkType.toLowerCase() === "image" || - hasImageFileExtension(assetUrl) - ); + // chunkType is sourced from Knowhere retrieval results, which can omit it + // at runtime even though the type says it's a required string. Normalize + // defensively instead of calling .toLowerCase() on a possibly-missing value. + const chunkType = + typeof citation.chunkType === "string" + ? citation.chunkType.toLowerCase() + : ""; + return chunkType === "image" || hasImageFileExtension(assetUrl); } function hasImageFileExtension(assetUrl: string): boolean { diff --git a/src/domains/chat/contracts.ts b/src/domains/chat/contracts.ts index 527b52c4..9accd573 100644 --- a/src/domains/chat/contracts.ts +++ b/src/domains/chat/contracts.ts @@ -16,7 +16,6 @@ import type { } from "@/domains/chat/types" import type { HardenMediaAssetUrls } from "./media-asset-hardening" import type { HardenChatAssetUrl } from "./media-assets" -import type { NotebookKnowhereRemoteDocumentClient } from "./knowhere-tools" export type RetrievalClient = { query(params: RetrievalQueryParams): Promise @@ -76,7 +75,6 @@ export type AnswerQuestionInput = { useAgentic?: boolean retrieval: RetrievalClient knowledge?: Knowledge - remoteDocumentClient?: NotebookKnowhereRemoteDocumentClient generateAnswer: GenerateAnswer hardenChatAssetUrl?: HardenChatAssetUrl hardenMediaAssetUrls?: HardenMediaAssetUrls diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index 021ebb22..b65be0f7 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -1,9 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest" import type { Knowledge, - KnowledgeGrepResponse, - KnowledgeOutline, - KnowledgeReadResponse, RetrievalQueryParams, RetrievalQueryResponse, RetrievalResult, @@ -150,7 +147,7 @@ describe("answerQuestionWithRetrieval", () => { }); }); - it("exposes search, list, outline, read, and grep through the Knowhere tool runtime", async () => { + it("exposes search through the Knowhere tool runtime", async () => { const result = makeRetrievalResult({ chunkType: "image", source: { @@ -170,36 +167,6 @@ describe("answerQuestionWithRetrieval", () => { answerText: null, }), }; - const getDocumentOutline = vi.fn().mockResolvedValue(makeKnowledgeOutline()); - const readChunks = vi.fn().mockResolvedValue( - makeKnowledgeReadResponse("Full diagram chunk body."), - ); - const grepChunks = vi.fn().mockResolvedValue(makeKnowledgeGrepResponse()); - const knowledge = { - getDocumentOutline, - readChunks, - grepChunks, - } as unknown as Knowledge; - const listDocuments = vi.fn().mockResolvedValue({ - documents: [ - { - documentId: "doc_remote", - namespace: "default", - status: "ready", - currentJobResultId: "job_remote", - sourceFileName: "remote.pdf", - documentMetadata: { - createdByClient: "cli", - }, - }, - { - documentId: "doc_untagged", - namespace: "default", - status: "ready", - sourceFileName: "dummy.pdf", - }, - ], - }); const generateAnswer = vi.fn( async ({ knowhereTools }: Parameters[0]) => { if (!knowhereTools) throw new Error("Knowhere tools were not provided."); @@ -209,28 +176,8 @@ describe("answerQuestionWithRetrieval", () => { targetContent: "image", topK: 2, }); - const documents = await knowhereTools.listDocuments(); - await knowhereTools.getDocumentOutline({ - documentId: "doc_included", - revisionKey: "job_123", - }); - await knowhereTools.readChunks({ - documentId: "doc_included", - revisionKey: "job_123", - page: 1, - pageSize: 2, - }); - await knowhereTools.grepChunks({ - documentId: "doc_included", - revisionKey: "job_123", - pattern: "diagram", - maxResults: 3, - }); expect(searchResponse.results).toEqual([result]); - expect( - documents.documents.map((document) => document.documentId), - ).toEqual(["doc_included", "doc_remote"]); return makeHarnessRunResult("Runtime answer."); }, ); @@ -246,8 +193,6 @@ describe("answerQuestionWithRetrieval", () => { sources, excludedSourceIds: ["source_excluded"], retrieval, - knowledge, - remoteDocumentClient: { documents: { list: listDocuments } }, generateAnswer, messages: [], }), @@ -261,27 +206,6 @@ describe("answerQuestionWithRetrieval", () => { dataType: 3, excludeDocumentIds: ["doc_excluded"], }); - expect(listDocuments).toHaveBeenCalledWith({ - namespace: "default", - page: 1, - pageSize: 200, - }); - expect(getDocumentOutline).toHaveBeenCalledWith({ - documentId: "doc_included", - revisionKey: "job_123", - }); - expect(readChunks).toHaveBeenCalledWith({ - documentId: "doc_included", - revisionKey: "job_123", - page: 1, - pageSize: 2, - }); - expect(grepChunks).toHaveBeenCalledWith({ - documentId: "doc_included", - revisionKey: "job_123", - pattern: "diagram", - maxResults: 3, - }); expect(answer.answer).toBe("Runtime answer."); }); @@ -1431,10 +1355,10 @@ describe("answerQuestionWithRetrieval", () => { expect(answer.citations[0]?.pageCitationAssetUrl).toBe(hardenedPageAssetUrl); }); - it("hydrates page numbers for grep citations from the matching parsed chunk", async () => { + it("hydrates page numbers for citations missing page metadata from the matching parsed chunk", async () => { const grepChunk = { - ref: "grep1:match:1", - kind: "grep_match" as const, + ref: "r1:result:1", + kind: "result" as const, chunkId: "chunk_financial_summary", content: "ept percentages and per share data)\nTotal automotive revenues\n17,693", contentPreview: "ept percentages and per share data)", @@ -1479,7 +1403,7 @@ describe("answerQuestionWithRetrieval", () => { makeHarnessRunResultWithLedger( "Automotive revenue was $17,693 million [[cite:1]].", { - citations: [{ ref: "grep1:match:1" }], + citations: [{ ref: "r1:result:1" }], chunks: [grepChunk], }, ), @@ -2054,17 +1978,7 @@ describe("answerQuestionWithRetrieval", () => { await tools.finalize?.execute({ text: "Information hiding is a module design principle.", - citations: [ - { - ref: "r1:result:1", - label: "claimed-source.pdf / Claimed", - source: { - documentId: "doc_claimed", - sourceFileName: "claimed-source.pdf", - sectionPath: "Claimed", - }, - }, - ], + citations: [{ pick: 1 }], memoryCitations: [], artifacts: [], unresolved: [], @@ -2698,17 +2612,7 @@ describe("generateAgenticOutputManifest", () => { }); await tools.finalize?.execute({ text: "已找到相关身份证图片，见下方图片。", - citations: [ - { - ref: "r1:result:1", - label: "商务标文件.pdf / 身份证正面", - source: { - documentId: "doc_identity", - sourceFileName: "document-generated.pdf", - sectionPath: "身份证正面", - }, - }, - ], + citations: [{ pick: 1 }], memoryCitations: [], artifacts: [ { @@ -2837,17 +2741,7 @@ describe("generateAgenticOutputManifest", () => { }); await tools.finalize?.execute({ text: "The inspected image appears to show the requested ID card.", - citations: [ - { - ref: "asset:r1:result:1", - label: "identity.pdf / images/id-front.png", - source: { - documentId: "doc_identity", - sourceFileName: "generated.pdf", - sectionPath: "images/id-front.png", - }, - }, - ], + citations: [{ pick: 1 }], memoryCitations: [], artifacts: [ { @@ -2926,7 +2820,7 @@ describe("generateAgenticOutputManifest", () => { ], }); expect(result.manifest.citations.map((citation) => citation.ref)).toEqual([ - "asset:r1:result:1", + "r1:result:1", ]); expect(result.manifest.artifacts).toEqual([ { @@ -2979,17 +2873,7 @@ describe("generateAgenticOutputManifest", () => { }); await tools.finalize?.execute({ text: "承包人自行修改发包人审批的进度计划，应按每次 5000 元赔偿违约金。", - citations: [ - { - ref: "asset:r1:referenced:1", - label: "投标书 / （6）现场工期进度管理方面的违约责任", - source: { - documentId: "doc_contract", - sourceFileName: null, - sectionPath: "Root / （6）现场工期进度管理方面的违约责任", - }, - }, - ], + citations: [{ pick: 1 }], memoryCitations: [], artifacts: [], unresolved: [], @@ -3088,7 +2972,7 @@ describe("generateAgenticOutputManifest", () => { }); expect(result.manifest.text).toContain("5000 元"); expect(result.manifest.citations.map((citation) => citation.ref)).toEqual([ - "asset:r1:referenced:1", + "r1:referenced:1", ]); expect(result.trace.toolCalls.map((call) => call.tool)).toContain( "inspectImage", @@ -3131,17 +3015,7 @@ describe("generateAgenticOutputManifest", () => { }); await tools.finalize?.execute({ text: "见下方图片。", - citations: [ - { - ref: "r1:result:1", - label: "ids.pdf / 身份证 1", - source: { - documentId: "doc_identity", - sourceFileName: "ids.pdf", - sectionPath: "身份证 1", - }, - }, - ], + citations: [{ pick: 1 }], memoryCitations: [], artifacts: [1, 2, 3].map((index) => ({ type: "image", @@ -3154,17 +3028,7 @@ describe("generateAgenticOutputManifest", () => { } else { await tools.finalize?.execute({ text: "见下方图片。", - citations: [ - { - ref: "r1:result:1", - label: "ids.pdf / 身份证 1", - source: { - documentId: "doc_identity", - sourceFileName: "ids.pdf", - sectionPath: "身份证 1", - }, - }, - ], + citations: [{ pick: 1 }], memoryCitations: [], artifacts: [1, 2].map((index) => ({ type: "image", @@ -3452,87 +3316,6 @@ function makeEvidenceChunkFromRetrievalResult( }; } -function makeKnowledgeOutline(): KnowledgeOutline { - return { - document: makeLocalKnowledgeDocument(), - totalChunks: 1, - typeCounts: { text: 1, image: 0, table: 0, page: 0 }, - sections: [ - { - sectionPath: "Root / Diagram", - sectionTitle: "Diagram", - sectionLevel: 2, - summary: "Diagram section.", - startChunk: 1, - endChunk: 1, - chunkCount: 1, - typeCounts: { text: 1, image: 0, table: 0, page: 0 }, - children: [], - }, - ], - sectionTree: [], - }; -} - -function makeKnowledgeReadResponse(content: string): KnowledgeReadResponse { - return { - document: makeLocalKnowledgeDocument(), - chunks: [ - { - position: 1, - chunkId: "chunk_1", - chunkType: "text", - content, - readableContent: content, - sectionPath: "Root / Diagram", - sourceChunkPath: "chunks/chunk-1.md", - filePath: "notes.txt", - metadata: {}, - }, - ], - page: 1, - pageSize: 1, - totalChunks: 1, - totalPages: 1, - }; -} - -function makeKnowledgeGrepResponse(): KnowledgeGrepResponse { - return { - document: makeLocalKnowledgeDocument(), - matches: [ - { - position: 1, - chunkId: "chunk_1", - chunkType: "text", - sectionPath: "Root / Diagram", - sourceChunkPath: "chunks/chunk-1.md", - filePath: "notes.txt", - startOffset: 0, - endOffset: 7, - snippet: "diagram", - }, - ], - scannedChunks: 1, - truncated: false, - }; -} - -function makeLocalKnowledgeDocument() { - return { - localDocumentId: "doc_included", - documentId: "doc_included", - jobId: "job_123", - namespace: "notebook-workspace", - sourceFileName: "notes.txt", - chunkCount: 1, - typeCounts: { text: 1, image: 0, table: 0, page: 0 }, - resultDirectoryPath: "parsed-storage:doc_included/job_123", - createdAt: new Date("2026-01-01T00:00:00Z"), - updatedAt: new Date("2026-01-01T00:00:00Z"), - }; -} - type KnowhereQueryResponseLogMeta = { readonly query: string readonly resultCount: number diff --git a/src/domains/chat/index.ts b/src/domains/chat/index.ts index 11a7b85c..75f1dafa 100644 --- a/src/domains/chat/index.ts +++ b/src/domains/chat/index.ts @@ -247,12 +247,7 @@ export const answerQuestionWithRetrieval = ( excludedSourceIds: input.excludedSourceIds, searchSources, knowhereTools: notebookKnowhereTools.createRuntime({ - namespace: input.namespace, - sources: input.sources, - excludedSourceIds: input.excludedSourceIds, searchSources, - knowledge: input.knowledge, - remoteDocumentClient: input.remoteDocumentClient, }), ...(input.inspectImages ? { inspectImages: input.inspectImages } : {}), }), @@ -1056,12 +1051,16 @@ function mapManifestCitationsToResults( ) const results: RetrievalResult[] = [] + const droppedRefs: string[] = [] for (const citation of result.manifest.citations) { const chunk = chunksByRef.get(citation.ref) ?? resolveChunkForAssetRef(citation.ref, assetsByRef, chunksByRef) - if (!chunk) continue + if (!chunk) { + droppedRefs.push(citation.ref) + continue + } const retrievalResult = toRetrievalResultFromEvidenceChunk( mergeChunkPageMetadata(chunk, result.trace.ledger.chunks), @@ -1080,6 +1079,14 @@ function mapManifestCitationsToResults( if (results.length >= MAX_CITATION_RESULTS) break } + if (droppedRefs.length > 0) { + logger.warn("chat-agent: dropped unresolved citation refs", { + droppedRefs, + ledgerChunkRefs: result.trace.ledger.chunks.map((chunk) => chunk.ref), + ledgerAssetRefs: result.trace.ledger.assets.map((asset) => asset.ref), + }) + } + return results } diff --git a/src/domains/chat/knowhere-tools.ts b/src/domains/chat/knowhere-tools.ts index 5c70fd7b..bac01f33 100644 --- a/src/domains/chat/knowhere-tools.ts +++ b/src/domains/chat/knowhere-tools.ts @@ -1,35 +1,7 @@ -import { Effect } from "effect" -import type { Knowledge } from "@ontos-ai/knowhere-sdk" - -import type { - KnowhereDocumentSummary, - KnowhereToolRuntime, -} from "@/agent-harness" -import type { Source } from "@/infrastructure/db/schema" -import { - listRemoteLibraryDocuments, - isNotebookVisibleRemoteDocument, - type RemoteLibraryDocument, -} from "@/domains/sources/remote-library" +import type { KnowhereToolRuntime } from "@/agent-harness" import type { SearchSources } from "./contracts" -import { excludeDocuments } from "./retrieval" - -type RemoteDocumentClient = Parameters< - typeof listRemoteLibraryDocuments ->[0]["client"] - -export type NotebookKnowhereRemoteDocumentClient = RemoteDocumentClient type NotebookKnowhereToolsInput = { - readonly namespace: string - readonly sources: readonly Source[] - readonly excludedSourceIds: readonly string[] - readonly searchSources: SearchSources - readonly knowledge?: Knowledge - readonly remoteDocumentClient?: RemoteDocumentClient -} - -type SearchOnlyRuntimeInput = { readonly searchSources: SearchSources } @@ -37,121 +9,6 @@ export const notebookKnowhereTools = { createRuntime(input: NotebookKnowhereToolsInput): KnowhereToolRuntime { return { search: (request) => input.searchSources(request), - listDocuments: async () => ({ - documents: await listVisibleDocuments(input), - }), - getDocumentOutline: async (request) => { - const knowledge = requireKnowledge(input.knowledge) - return knowledge.getDocumentOutline(request) - }, - readChunks: async (request) => { - const knowledge = requireKnowledge(input.knowledge) - return knowledge.readChunks(request) - }, - grepChunks: async (request) => { - const knowledge = requireKnowledge(input.knowledge) - return knowledge.grepChunks(request) - }, - } - }, - - createSearchOnlyRuntime(input: SearchOnlyRuntimeInput): KnowhereToolRuntime { - return { - search: (request) => input.searchSources(request), - listDocuments: async () => ({ documents: [] }), - getDocumentOutline: async () => { - throw new Error("Knowhere document outline is not configured.") - }, - readChunks: async () => { - throw new Error("Knowhere chunk reads are not configured.") - }, - grepChunks: async () => { - throw new Error("Knowhere grep is not configured.") - }, } }, } as const - -async function listVisibleDocuments( - input: NotebookKnowhereToolsInput, -): Promise { - const excludedSourceIds = new Set(input.excludedSourceIds) - const excludedDocumentIds = new Set( - excludeDocuments(input.sources, input.excludedSourceIds) - .excludeDocumentIds ?? [], - ) - const localDocuments = input.sources - .filter( - (source): source is Source & { readonly knowhereDocumentId: string } => - source.status === "ready" && - Boolean(source.knowhereDocumentId) && - !excludedSourceIds.has(source.id) && - !excludedDocumentIds.has(source.knowhereDocumentId ?? ""), - ) - .map((source): KnowhereDocumentSummary => ({ - documentId: source.knowhereDocumentId, - revisionKey: source.knowhereJobId ?? undefined, - namespace: input.namespace, - sourceFileName: source.title, - title: source.title, - status: source.status, - })) - - const remoteDocuments = await listVisibleRemoteDocuments({ - input, - localDocuments, - excludedDocumentIds, - }) - - return [...localDocuments, ...remoteDocuments] -} - -async function listVisibleRemoteDocuments(input: { - readonly input: NotebookKnowhereToolsInput - readonly localDocuments: readonly KnowhereDocumentSummary[] - readonly excludedDocumentIds: ReadonlySet -}): Promise { - if (!input.input.remoteDocumentClient) return [] - - const localDocumentIds = new Set( - input.localDocuments.flatMap((document): string[] => - document.documentId ? [document.documentId] : [], - ), - ) - const documents = await Effect.runPromise( - listRemoteLibraryDocuments({ - workspace: { namespace: input.input.namespace }, - client: input.input.remoteDocumentClient, - localSources: input.input.sources, - }), - ) - - return documents - .filter( - (document) => - isNotebookVisibleRemoteDocument(document) && - document.status === "ready" && - !localDocumentIds.has(document.documentId) && - !input.excludedDocumentIds.has(document.documentId), - ) - .map(toRemoteDocumentSummary) -} - -function toRemoteDocumentSummary( - document: RemoteLibraryDocument, -): KnowhereDocumentSummary { - return { - documentId: document.documentId, - revisionKey: document.revisionKey, - namespace: document.namespace, - sourceFileName: - document.sourceFileName ?? document.title ?? document.documentId, - title: document.title, - status: document.status, - } -} - -function requireKnowledge(knowledge: Knowledge | undefined): Knowledge { - if (knowledge) return knowledge - throw new Error("Knowhere parsed-document reads are not configured.") -} diff --git a/src/domains/chat/media-assets.test.ts b/src/domains/chat/media-assets.test.ts index 1d744826..c4ee72f5 100644 --- a/src/domains/chat/media-assets.test.ts +++ b/src/domains/chat/media-assets.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest" import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" import { + dedupeMediaCitationResults, enrichRetrievalResultsWithAssetUrls, formatRetrievedMediaAssetContext, isImageAssetUrl, @@ -9,6 +10,14 @@ import { } from "./media-assets" import type { Source } from "@/infrastructure/db/schema" +vi.mock("@/lib/logger", () => ({ + logger: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})) + describe("chat media assets", () => { it("enriches retrieved image chunks from Notebook parsed asset URLs", async () => { const hardenChatAssetUrl = vi @@ -259,6 +268,22 @@ describe("chat media assets", () => { expect(answer).toBe("{\"name\":\"冯荣洲\",\"status\":\"matched\"}") }) + + it("dedupes results with a missing chunkType instead of throwing", () => { + // Knowhere's chunkType is declared as a required string in the SDK + // type, but real retrieval results can omit it at runtime. + const resultWithoutChunkType = makeRetrievalResult({ + chunkType: undefined as unknown as string, + assetUrl: "https://blob.example/images/launch.jpg", + }) + + expect(() => + dedupeMediaCitationResults([resultWithoutChunkType]), + ).not.toThrow() + + const deduped = dedupeMediaCitationResults([resultWithoutChunkType]) + expect(deduped).toEqual([resultWithoutChunkType]) + }) }) function makeRetrievalResult( diff --git a/src/domains/chat/media-assets.ts b/src/domains/chat/media-assets.ts index 423a8788..09c9cd07 100644 --- a/src/domains/chat/media-assets.ts +++ b/src/domains/chat/media-assets.ts @@ -1,7 +1,15 @@ import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" +import { logger } from "@/lib/logger" import type { Source } from "@/infrastructure/db/schema" +// Knowhere's chunkType is declared as a required string in the SDK type, +// but real retrieval results can omit it. Normalize defensively instead of +// calling .toLowerCase() on a value that may be undefined at runtime. +function normalizeChunkType(chunkType: string): string { + return typeof chunkType === "string" ? chunkType.toLowerCase() : "" +} + const retrievedMediaAssetLimit = 6 const imageExtensions = [".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg"] as const const internalMetadataKeys = new Set([ @@ -79,7 +87,7 @@ export function dedupeMediaCitationResults( const existingResult = dedupedResults[existingIndex] const existingAssetUrl = getTrimmedString(existingResult?.assetUrl) - if ( + const keepCurrent = existingResult && existingAssetUrl && compareMediaCitationResult( @@ -88,7 +96,16 @@ export function dedupeMediaCitationResults( assetUrl, existingAssetUrl, ) > 0 - ) { + logger.info("chat-agent: merged duplicate media citation", { + assetKey, + keptChunkId: keepCurrent + ? (result.chunkId ?? null) + : (existingResult?.chunkId ?? null), + droppedChunkId: keepCurrent + ? (existingResult?.chunkId ?? null) + : (result.chunkId ?? null), + }) + if (keepCurrent) { dedupedResults[existingIndex] = result } } @@ -161,7 +178,7 @@ function getMediaCitationResultScore( result: RetrievalResult, assetUrl: string, ): number { - const chunkType = result.chunkType.toLowerCase() + const chunkType = normalizeChunkType(result.chunkType) const isImageAsset = isImageAssetUrl(assetUrl) const isTableAsset = chunkType === "table" const source = result.source @@ -234,7 +251,7 @@ async function addAssetCitationResults( source: Source, hardenChatAssetUrl: HardenChatAssetUrl, ): Promise { - if (result.chunkType.toLowerCase() === "page") return [result] + if (normalizeChunkType(result.chunkType) === "page") return [result] const existingAssetUrl = getTrimmedString(result.assetUrl) if (existingAssetUrl && isNotebookOwnedAssetUrl(existingAssetUrl)) return [result] @@ -433,7 +450,7 @@ function isRenderableMediaAsset( result: RetrievalResult, assetUrl: string, ): boolean { - const chunkType = result.chunkType.toLowerCase() + const chunkType = normalizeChunkType(result.chunkType) return chunkType === "image" || chunkType === "table" || isImageAssetUrl(assetUrl) } diff --git a/src/domains/chat/page-citation-assets.test.ts b/src/domains/chat/page-citation-assets.test.ts index 5b162641..06f679ed 100644 --- a/src/domains/chat/page-citation-assets.test.ts +++ b/src/domains/chat/page-citation-assets.test.ts @@ -203,6 +203,31 @@ describe("enrichRetrievalResultsWithPageCitationAssetUrls", () => { expect(result?.pageCitationAssetUrl).toBeUndefined() expect(result?.pageCitationPageNumber).toBe(4) }) + + it("does not throw when a result is missing chunkType", async () => { + // Knowhere's chunkType is declared as a required string in the SDK + // type, but real retrieval results can omit it at runtime. + const resultWithoutChunkType = makeRetrievalResult({ + chunkType: undefined as unknown as string, + metadata: { + pageNums: [4], + pageAssets: [ + { + pageNum: 4, + artifactRef: "page_citation_assets/page-4.png", + assetUrl: "https://assets.example/pages/page-4.png", + }, + ], + }, + }) + + const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ + results: [resultWithoutChunkType], + sources: [makeSource()], + }) + + expect(result?.pageCitationAssetUrl).toBeUndefined() + }) }) function makeRetrievalResult( diff --git a/src/domains/chat/page-citation-assets.ts b/src/domains/chat/page-citation-assets.ts index 3007f60d..07e93197 100644 --- a/src/domains/chat/page-citation-assets.ts +++ b/src/domains/chat/page-citation-assets.ts @@ -122,7 +122,12 @@ async function getStoredPageCitationAssetUrl(input: { } function isPageResult(result: RetrievalResult): boolean { - return result.chunkType.toLowerCase() === "page" + // Knowhere's chunkType is declared as a required string in the SDK type, + // but real retrieval results can omit it. Normalize defensively instead + // of calling .toLowerCase() on a value that may be undefined at runtime. + return typeof result.chunkType === "string" + ? result.chunkType.toLowerCase() === "page" + : false } function getDirectPageCitationAsset( diff --git a/src/domains/chat/prompt.ts b/src/domains/chat/prompt.ts index 4f721019..00f3eadf 100644 --- a/src/domains/chat/prompt.ts +++ b/src/domains/chat/prompt.ts @@ -62,7 +62,7 @@ export const generateAgenticOutputManifestEffect = ( turn, knowhereTools: input.knowhereTools ?? - notebookKnowhereTools.createSearchOnlyRuntime({ + notebookKnowhereTools.createRuntime({ searchSources: input.searchSources, }), memoryTools: mementoMemoryTools.createRuntime({ diff --git a/src/domains/chat/route-answer.ts b/src/domains/chat/route-answer.ts index 0a7cbb4d..34cc4d7d 100644 --- a/src/domains/chat/route-answer.ts +++ b/src/domains/chat/route-answer.ts @@ -145,7 +145,6 @@ const answerChatEffect = (input: AnswerChatInput) => excludedSourceIds: body.value.excludedSourceIds, retrieval: client.retrieval, knowledge: knowhereResources.knowledge, - remoteDocumentClient: client, hardenChatAssetUrl, hardenMediaAssetUrls: ({ results, artifacts }) => hardenChatMediaAssetUrls({ diff --git a/src/domains/chat/service.ts b/src/domains/chat/service.ts index 7839ab02..8139a42f 100644 --- a/src/domains/chat/service.ts +++ b/src/domains/chat/service.ts @@ -68,7 +68,6 @@ type ChatTurnInput = { excludedSourceIds: readonly string[] retrieval: RetrievalClient knowledge?: AnswerQuestionInput["knowledge"] - remoteDocumentClient?: AnswerQuestionInput["remoteDocumentClient"] generateAnswer: GenerateAnswer hardenChatAssetUrl?: AnswerQuestionInput["hardenChatAssetUrl"] hardenMediaAssetUrls?: AnswerQuestionInput["hardenMediaAssetUrls"] @@ -131,7 +130,6 @@ export const handleChatTurnEffect = (input: ChatTurnInput) => excludedSourceIds: input.excludedSourceIds, retrieval: input.retrieval, knowledge: input.knowledge, - remoteDocumentClient: input.remoteDocumentClient, generateAnswer: input.generateAnswer, hardenChatAssetUrl: input.hardenChatAssetUrl, hardenMediaAssetUrls: input.hardenMediaAssetUrls, From 568587f4cb4f53edbbd348999cdc55b1938f18f3 Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Fri, 11 Sep 2026 16:05:24 +0800 Subject: [PATCH 11/11] fix(chat): honor document scope and block source-required finalize before search Pass include/exclude document IDs through to Knowhere and keep finalize closed until a required document search has run. Co-authored-by: Cursor --- src/agent-harness/runtime.test.ts | 53 ++++++++++- src/agent-harness/runtime.ts | 38 +++++++- src/agent-harness/types.ts | 3 + src/domains/chat/contracts.ts | 9 +- src/domains/chat/index.test.ts | 91 +++++++++++++++++++ src/domains/chat/index.ts | 5 +- src/domains/chat/knowhere-tools.ts | 27 +++++- src/domains/chat/prompt.ts | 1 + src/domains/chat/retrieval.ts | 20 +++- .../knowhere-retrieval-wire.test.ts | 90 ++++++++++++++++++ 10 files changed, 320 insertions(+), 17 deletions(-) create mode 100644 src/integrations/knowhere-retrieval-wire.test.ts diff --git a/src/agent-harness/runtime.test.ts b/src/agent-harness/runtime.test.ts index 11ac3d94..d36ca1be 100644 --- a/src/agent-harness/runtime.test.ts +++ b/src/agent-harness/runtime.test.ts @@ -124,6 +124,19 @@ describe("agent harness runtime", () => { ]) }) + it.each([ + { includeDocumentIds: ["doc_1"], excludeDocumentIds: ["doc_2"] }, + { includeDocumentIds: [], excludeDocumentIds: [] }, + ])("preserves document scopes in the actual search tool: %j", async (scope) => { + const search = vi.fn().mockResolvedValue(makeRetrievalResponse()) + const tools = createHarnessTools({ + state: {}, ledger: createEvidenceLedger(), recentTurns: [], + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(search), + }) + await executeTool(tools.knowhere_search, { query: "question", ...scope }) + expect(search).toHaveBeenCalledWith(expect.objectContaining(scope)) + }) + it("returns only newly searched evidence in each knowhere_search tool result", async () => { const query = vi .fn() @@ -1083,8 +1096,8 @@ describe("agent harness runtime", () => { "setContextPolicy", "inspectImage", "readPriorTurn", - "finalize", ]) + expect(result.activeTools).not.toContain("finalize") expect(result.activeTools).not.toContain("memory_search") expect(result.activeTools).not.toContain("knowhere_search") }) @@ -1104,6 +1117,7 @@ describe("agent harness runtime", () => { }) expect(result.activeTools).toContain("memory_search") + expect(result.activeTools).toContain("finalize") expect(result.activeTools).not.toContain("knowhere_search") }) @@ -1121,6 +1135,7 @@ describe("agent harness runtime", () => { messages: [], }) + expect(result.activeTools).toContain("finalize") expect(result.activeTools).not.toContain("memory_search") expect(result.activeTools).not.toContain("knowhere_search") }) @@ -1132,6 +1147,22 @@ describe("agent harness runtime", () => { }) expect(result.toolChoice).toBe("required") + expect(result.activeTools).not.toContain("finalize") + }) + + it("blocks finalize on the first step so a document question cannot skip search", () => { + const result = prepareHarnessStep({ + stepNumber: 1, + messages: [ + { + role: "user", + content: "高血压合并冠心病，血压目标一般怎么定？", + }, + ], + }) + + expect(result.activeTools).not.toContain("finalize") + expect(result.toolChoice).toBe("required") }) it("opens memory_search and knowhere_search together as peers when sources are required", () => { @@ -1151,12 +1182,32 @@ describe("agent harness runtime", () => { expect(result.activeTools).toEqual( expect.arrayContaining(["memory_search", "knowhere_search"]), ) + expect(result.activeTools).not.toContain("finalize") expect(result.activeTools).not.toContain("knowhere_list_documents") expect(result.activeTools).not.toContain("knowhere_get_document_outline") expect(result.activeTools).not.toContain("knowhere_read_chunks") expect(result.activeTools).not.toContain("knowhere_grep_chunks") }) + it("reopens finalize after must_use_sources has called knowhere_search, including empty results", () => { + const result = prepareHarnessStep({ + stepNumber: 4, + hasKnowhereSearch: true, + intent: { + task: "answer", + dependsOnPreviousTurn: false, + retrievalNeeded: "yes", + targetModalities: ["text"], + constraints: {}, + groundingPolicy: "must_use_sources", + }, + messages: [], + }) + + expect(result.activeTools).toContain("finalize") + expect(result.activeTools).toContain("knowhere_search") + }) + it("forces image inspection before forced finalization when image assets are available", () => { const result = prepareHarnessStep({ stepNumber: 12, diff --git a/src/agent-harness/runtime.ts b/src/agent-harness/runtime.ts index 43d9329a..dbe91a58 100644 --- a/src/agent-harness/runtime.ts +++ b/src/agent-harness/runtime.ts @@ -90,6 +90,12 @@ const knowhereSearchTargetContentSchema = z.enum([ const knowhereSearchSchema = z.object({ query: z.string().min(1), + includeDocumentIds: z.array(z.string().trim().min(1)).optional().describe( + "Only search these verified document IDs. Omit for unrestricted search; [] searches no documents. Use IDs from source context or previous search results, never filenames or guessed IDs.", + ), + excludeDocumentIds: z.array(z.string().trim().min(1)).optional().describe( + "Exclude these verified document IDs. Exclusions take precedence over includeDocumentIds. If the ID is unknown, describe the document constraint in query instead.", + ), targetContent: knowhereSearchTargetContentSchema.default("all"), purpose: z.string().optional(), topK: z.number().int().min(1).max(12).optional(), @@ -215,6 +221,9 @@ export async function runAgentHarness( hasUninspectedImageAssets: input.inspectImages !== undefined && hasUninspectedImageAssets({ state, ledger }), + hasKnowhereSearch: (state.toolCalls ?? []).some( + (call) => call.tool === "knowhere_search", + ), }), stopWhen: [ () => state.finalized === true, @@ -249,7 +258,6 @@ const alwaysAvailableTools = [ "setContextPolicy", "inspectImage", "readPriorTurn", - "finalize", ] as const const fluidRetrievalTools = ["memory_search"] as const @@ -273,6 +281,7 @@ export function prepareHarnessStep(input: { readonly stepNumber: number readonly messages: readonly ModelMessage[] readonly hasUninspectedImageAssets?: boolean + readonly hasKnowhereSearch?: boolean readonly intent?: IntentFrame }): HarnessStepPreparation { const messages = sanitizeHarnessModelMessagesForStep(input.messages) @@ -319,6 +328,7 @@ export function prepareHarnessStep(input: { messages, activeTools: selectHarnessActiveTools({ intent: input.intent, + hasKnowhereSearch: input.hasKnowhereSearch === true, }), // finalize is the only output contract (see its tool description). Force // a tool call every step so the model cannot end the turn with a bare @@ -329,10 +339,14 @@ export function prepareHarnessStep(input: { function selectHarnessActiveTools(input: { readonly intent?: IntentFrame + readonly hasKnowhereSearch: boolean }): Array> { const tools: Array> = [ ...alwaysAvailableTools, ] + if (allowsFinalize(input)) { + tools.push("finalize") + } if (!allowsRetrieval(input.intent)) { return tools } @@ -355,6 +369,21 @@ function allowsRetrieval(intent?: IntentFrame): boolean { ) } +function allowsFinalize(input: { + readonly intent?: IntentFrame + readonly hasKnowhereSearch: boolean +}): boolean { + if (!input.intent) return false + if ( + input.intent.groundingPolicy === "must_use_sources" && + allowsRetrieval(input.intent) && + !input.hasKnowhereSearch + ) { + return false + } + return true +} + export function sanitizeHarnessModelMessagesForStep( messages: readonly ModelMessage[], ): ModelMessage[] { @@ -1001,6 +1030,12 @@ async function executeKnowhereSearch(input: { const beforeSnapshot = input.ledger.snapshot() const response = await input.knowhereTools.search({ query: input.request.query, + ...(input.request.includeDocumentIds !== undefined + ? { includeDocumentIds: input.request.includeDocumentIds } + : {}), + ...(input.request.excludeDocumentIds !== undefined + ? { excludeDocumentIds: input.request.excludeDocumentIds } + : {}), targetContent: input.request.targetContent, purpose: input.request.purpose, topK: input.request.topK, @@ -1280,6 +1315,7 @@ export function buildHarnessSystemPrompt(turn: AgentTurnInput): string { "- Call knowhere_search only when memory is insufficient and groundingPolicy requires citing source documents.", "- Refine knowhere_search at most twice. If two refined searches still do not add new relevant evidence, call finalize and list the gap in unresolved.", "- Do not treat every question as a document-retrieval task.", + "- For document-scoped searches, use includeDocumentIds/excludeDocumentIds only with verified IDs from source context or prior search results. If IDs are unknown, preserve the document requirement in query so Knowhere can locate it. Never invent IDs or substitute filenames. Exclusions win; an empty includeDocumentIds means no documents.", "", "Context rules:", "- If the current user request is unrelated to prior turns, set carryHistory to none and do not reuse prior topics.", diff --git a/src/agent-harness/types.ts b/src/agent-harness/types.ts index ad686ad6..b5a94168 100644 --- a/src/agent-harness/types.ts +++ b/src/agent-harness/types.ts @@ -89,6 +89,9 @@ export type KnowhereSearchRequest = Pick< RetrievalQueryParams, "query" | "topK" | "signalPaths" | "filterMode" | "threshold" > & { + /** Omitted means all documents; [] means none. Exclusions take precedence. */ + readonly includeDocumentIds?: string[] + readonly excludeDocumentIds?: string[] readonly targetContent?: KnowhereSearchTargetContent readonly purpose?: string } diff --git a/src/domains/chat/contracts.ts b/src/domains/chat/contracts.ts index 9accd573..a38a6f98 100644 --- a/src/domains/chat/contracts.ts +++ b/src/domains/chat/contracts.ts @@ -9,6 +9,7 @@ import type { HarnessRunResult, InspectImages, KnowhereToolRuntime, + KnowhereSearchRequest, } from "@/agent-harness" import type { ChatArtifactView, @@ -40,13 +41,7 @@ export type AgenticRetrievalPlan = { purpose: string | null } -export type AgenticRetrievalQuery = Pick< - RetrievalQueryParams, - "query" | "topK" | "signalPaths" | "filterMode" | "threshold" -> & { - readonly targetContent?: AgenticRetrievalTargetContent - readonly purpose?: string -} +export type AgenticRetrievalQuery = KnowhereSearchRequest export type AgenticRetrievalResponse = RetrievalQueryResponse & { retrievalPlan?: AgenticRetrievalPlan diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index b65be0f7..26fd0385 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -1,4 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest" +import { createServer } from "node:http" +import { once } from "node:events" +import Knowhere from "@ontos-ai/knowhere-sdk" import type { Knowledge, RetrievalQueryParams, @@ -103,6 +106,94 @@ describe("answerQuestionWithRetrieval", () => { }); }); + it.each([true, false])("sends document scope through the installed SDK (agentic=%s)", async (useAgentic) => { + const received: Record[] = [] + const result = makeRetrievalResult({ + content: "Evidence from the allowed document.", + source: { documentId: "doc_included", sourceFileName: "notes.txt", sectionPath: "Overview" }, + }) + const server = createServer((request, response) => { + let body = "" + request.setEncoding("utf8") + request.on("data", (chunk: string) => { body += chunk }) + request.on("end", () => { + received.push(JSON.parse(body)) + response.writeHead(200, { "Content-Type": "application/json" }) + response.end(JSON.stringify({ + namespace: "default", query: "document question", router_used: "test_fixture", + results: [{ content: result.content, chunk_type: "text", score: 1, + source: { document_id: "doc_included", source_file_name: "notes.txt", section_path: "Overview" } }], + referenced_chunks: [], evidence_text: result.content, answer_text: "", + })) + }) + }) + try { + server.listen(0, "127.0.0.1") + await once(server, "listening") + const address = server.address() + if (!address || typeof address === "string") throw new Error("Expected TCP address") + const client = new Knowhere({ apiKey: "scope-test", baseURL: `http://127.0.0.1:${address.port}`, maxRetries: 0 }) + const scopes = [ + {}, + { includeDocumentIds: ["doc_included", "doc_other"] }, + { excludeDocumentIds: ["doc_other"] }, + { includeDocumentIds: ["doc_included", "doc_other"], excludeDocumentIds: ["doc_other"] }, + { includeDocumentIds: [] }, + ] + const answer = await Effect.runPromise(answerQuestionWithRetrieval({ + question: "document question", namespace: "default", useAgentic, + sources: [makeSource(), makeSource({ id: "source_other", knowhereDocumentId: "doc_other" })], + excludedSourceIds: ["knowhere-doc:default:doc_user_excluded"], + retrieval: client.retrieval, + generateAnswer: async ({ knowhereTools }) => { + if (!knowhereTools) throw new Error("Missing Knowhere tools") + for (const scope of scopes) await knowhereTools.search({ query: "document question", ...scope }) + return makeCitedHarnessRunResult("Evidence [[cite:1]].", result) + }, + messages: [], + })) + expect(received).toEqual(scopes.map((scope) => ({ + namespace: "default", query: "document question", top_k: 8, use_agentic: useAgentic, data_type: 1, + ...(scope.includeDocumentIds !== undefined ? { include_document_ids: scope.includeDocumentIds } : {}), + exclude_document_ids: ["doc_user_excluded", ...(scope.excludeDocumentIds ?? [])], + }))) + expect(answer.answer).toBe("Evidence [[cite:1]].") + expect(answer.citations).toHaveLength(1) + expect(answer.citations[0]?.source.documentId).toBe("doc_included") + } finally { + if (server.listening) { + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()) + server.closeAllConnections() + }) + } + } + }) + + it("rejects invented document IDs but accepts IDs discovered by a prior search", async () => { + const result = makeRetrievalResult({ source: { + documentId: "doc_discovered", sourceFileName: "named-document.pdf", sectionPath: "Overview", + } }) + const retrieval = { query: vi.fn().mockResolvedValue({ + namespace: "default", query: "named document", routerUsed: "agent_explore", + results: [result], referencedChunks: [], evidenceText: result.content, answerText: "", + }) } + await Effect.runPromise(answerQuestionWithRetrieval({ + question: "Only search the named document", namespace: "default", + sources: [makeSource()], excludedSourceIds: [], retrieval, messages: [], + generateAnswer: async ({ knowhereTools }) => { + if (!knowhereTools) throw new Error("Missing Knowhere tools") + await expect(knowhereTools.search({ query: "question", includeDocumentIds: ["named-document.pdf"] })).rejects.toThrow("unverified ID") + await expect(knowhereTools.search({ query: "question", excludeDocumentIds: ["doc_invented"] })).rejects.toThrow("unverified ID") + expect(retrieval.query).not.toHaveBeenCalled() + await knowhereTools.search({ query: "Only search named-document.pdf" }) + await knowhereTools.search({ query: "question", includeDocumentIds: ["doc_discovered"] }) + expect(retrieval.query).toHaveBeenLastCalledWith(expect.objectContaining({ includeDocumentIds: ["doc_discovered"] })) + return makeCitedHarnessRunResult("Evidence [[cite:1]].", result) + }, + })) + }) + it("does not create source chips from retrieval results when the manifest has no citations", async () => { const unrelatedResult = makeRetrievalResult({ content: "Information hiding is unrelated to the requested source.", diff --git a/src/domains/chat/index.ts b/src/domains/chat/index.ts index 75f1dafa..9f73b20b 100644 --- a/src/domains/chat/index.ts +++ b/src/domains/chat/index.ts @@ -32,7 +32,7 @@ import type { AnswerQuestionResult, } from "./contracts" import { - excludeDocuments, + getRetrievalDocumentScope, normalizeRetrievalQuery, } from "./retrieval" import { @@ -248,6 +248,7 @@ export const answerQuestionWithRetrieval = ( searchSources, knowhereTools: notebookKnowhereTools.createRuntime({ searchSources, + sources: input.sources, }), ...(input.inspectImages ? { inspectImages: input.inspectImages } : {}), }), @@ -976,7 +977,7 @@ function buildRetrievalQueryParams(input: { ...(typeof input.input.threshold === "number" ? { threshold: input.input.threshold } : {}), - ...excludeDocuments(input.sources, input.excludedSourceIds), + ...getRetrievalDocumentScope(input.sources, input.excludedSourceIds, input.input), } } diff --git a/src/domains/chat/knowhere-tools.ts b/src/domains/chat/knowhere-tools.ts index bac01f33..49bdbb92 100644 --- a/src/domains/chat/knowhere-tools.ts +++ b/src/domains/chat/knowhere-tools.ts @@ -1,14 +1,39 @@ import type { KnowhereToolRuntime } from "@/agent-harness" import type { SearchSources } from "./contracts" +import type { Source } from "@/infrastructure/db/schema" type NotebookKnowhereToolsInput = { readonly searchSources: SearchSources + readonly sources: readonly Source[] } export const notebookKnowhereTools = { createRuntime(input: NotebookKnowhereToolsInput): KnowhereToolRuntime { + const knownDocumentIds = new Set( + input.sources.flatMap((source) => + source.knowhereDocumentId ? [source.knowhereDocumentId] : [], + ), + ) return { - search: (request) => input.searchSources(request), + search: async (request) => { + const requestedIds = [ + ...(request.includeDocumentIds ?? []), + ...(request.excludeDocumentIds ?? []), + ] + if (requestedIds.some((id) => !knownDocumentIds.has(id))) { + throw new Error( + "Document scope contains an unverified ID. Use document IDs from source context or prior search results; otherwise keep the document requirement in query so Knowhere can locate it.", + ) + } + const response = await input.searchSources(request) + for (const result of response.results) { + if (result.source.documentId) knownDocumentIds.add(result.source.documentId) + } + for (const ref of response.referencedChunks) { + if (ref.documentId) knownDocumentIds.add(ref.documentId) + } + return response + }, } }, } as const diff --git a/src/domains/chat/prompt.ts b/src/domains/chat/prompt.ts index 00f3eadf..4d24c99a 100644 --- a/src/domains/chat/prompt.ts +++ b/src/domains/chat/prompt.ts @@ -64,6 +64,7 @@ export const generateAgenticOutputManifestEffect = ( input.knowhereTools ?? notebookKnowhereTools.createRuntime({ searchSources: input.searchSources, + sources: input.sources, }), memoryTools: mementoMemoryTools.createRuntime({ workspaceId: input.workspaceId, diff --git a/src/domains/chat/retrieval.ts b/src/domains/chat/retrieval.ts index 4d1103fa..e5391136 100644 --- a/src/domains/chat/retrieval.ts +++ b/src/domains/chat/retrieval.ts @@ -1,4 +1,4 @@ -import type { RetrievalQueryParams } from "@ontos-ai/knowhere-sdk" +import type { KnowhereSearchRequest } from "@/agent-harness/types" import type { Source } from "@/infrastructure/db/schema" import { decodeRemoteSourceId } from "@/domains/sources/remote-library" @@ -21,10 +21,11 @@ export function normalizeRetrievalQuery(value: string, fallback: string): string return normalized.slice(0, RETRIEVAL_QUERY_CHAR_LIMIT) } -export function excludeDocuments( +export function getRetrievalDocumentScope( sources: readonly Source[], excludedSourceIds: readonly string[], -): Pick { + request: Pick, +): Pick { const excluded = new Set(excludedSourceIds) const localDocumentIds = sources .filter((source) => excluded.has(source.id)) @@ -34,10 +35,19 @@ export function excludeDocuments( .map((sourceId) => decodeRemoteSourceId(sourceId)?.documentId) .filter((documentId): documentId is string => Boolean(documentId)) const documentIds = Array.from( - new Set([...localDocumentIds, ...remoteDocumentIds]), + new Set([ + ...localDocumentIds, + ...remoteDocumentIds, + ...(request.excludeDocumentIds ?? []), + ]), ) - return documentIds.length > 0 ? { excludeDocumentIds: documentIds } : {} + return { + ...(request.includeDocumentIds !== undefined + ? { includeDocumentIds: [...new Set(request.includeDocumentIds)] } + : {}), + ...(documentIds.length > 0 ? { excludeDocumentIds: documentIds } : {}), + } } function stripWrappingQuotes(value: string): string { diff --git a/src/integrations/knowhere-retrieval-wire.test.ts b/src/integrations/knowhere-retrieval-wire.test.ts new file mode 100644 index 00000000..cf698a2e --- /dev/null +++ b/src/integrations/knowhere-retrieval-wire.test.ts @@ -0,0 +1,90 @@ +import { createServer } from "node:http" +import { once } from "node:events" +import { afterEach, describe, expect, it } from "vitest" +import type { RetrievalQueryParams } from "@ontos-ai/knowhere-sdk" + +import { makeKnowhereClient } from "./knowhere" + +describe("makeKnowhereClient retrieval wire payload", () => { + const originalBaseURL = process.env.KNOWHERE_BASE_URL + + afterEach(() => { + restoreEnv("KNOWHERE_BASE_URL", originalBaseURL) + }) + + it.each<{ + name: string + params: RetrievalQueryParams + body: string + }>([ + { + name: "serializes overlapping include and exclude IDs", + params: { + query: "battery charging", + includeDocumentIds: ["doc_123", "doc_old"], + excludeDocumentIds: ["doc_old"], + } as RetrievalQueryParams, + body: '{"query":"battery charging","include_document_ids":["doc_123","doc_old"],"exclude_document_ids":["doc_old"]}', + }, + { + name: "preserves empty inclusion and exclusion arrays", + params: { + query: "battery charging", + includeDocumentIds: [], + excludeDocumentIds: [], + } as RetrievalQueryParams, + body: '{"query":"battery charging","include_document_ids":[],"exclude_document_ids":[]}', + }, + { + name: "omits document filters when not supplied", + params: { query: "battery charging" }, + body: '{"query":"battery charging"}', + }, + ])("$name", async ({ params, body }) => { + let receivedBody = "" + let receivedMethod: string | undefined + let receivedUrl: string | undefined + const server = createServer((request, response) => { + receivedMethod = request.method + receivedUrl = request.url + request.setEncoding("utf8") + request.on("data", (chunk: string) => { + receivedBody += chunk + }) + request.on("end", () => { + response.writeHead(200, { "Content-Type": "application/json" }) + response.end('{"results":[]}') + }) + }) + + try { + server.listen(0, "127.0.0.1") + await once(server, "listening") + const address = server.address() + if (!address || typeof address === "string") { + throw new Error("Expected TCP server address") + } + process.env.KNOWHERE_BASE_URL = `http://127.0.0.1:${address.port}` + const client = makeKnowhereClient("scope-test") + + await client.retrieval.query(params) + + expect(receivedMethod).toBe("POST") + expect(receivedUrl).toBe("/v2/retrieval/query") + expect(receivedBody).toBe(body) + } finally { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())) + server.closeAllConnections() + }) + } + }) +}) + +function restoreEnv(key: string, value: string | undefined): void { + if (value === undefined) { + delete process.env[key] + return + } + process.env[key] = value +}