From 02151b2da9a0814223ec62127a92350a8631f8a9 Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Wed, 9 Sep 2026 14:45:48 +0800 Subject: [PATCH 1/4] feat(memory): implement coarse capture of user observations - Introduced a new observation layer for capturing user insights without early classification. - Updated the memory extraction workflow to persist observations and trigger distillation. - Refactored related services and repository methods to support the new observation model. - Added tests for the new capture prompt and ensured existing functionality remains intact. This change enhances the ability to gather user concerns and insights for future processing. --- .pnpm-store/v11/index.db | Bin 0 -> 8192 bytes .repos/OpenViking | 1 + drizzle/0015_bumpy_vulcan.sql | 17 + drizzle/meta/0015_snapshot.json | 1444 +++++++++++++++++ drizzle/meta/_journal.json | 7 + pnpm-workspace.yaml | 8 + src/app/api/memory/distill/route.ts | 27 + src/domains/memory/distill-config.ts | 19 + src/domains/memory/distill-model.ts | 83 + src/domains/memory/distill-prompts.test.ts | 110 ++ src/domains/memory/distill-prompts.ts | 311 ++++ src/domains/memory/distill-trigger.test.ts | 84 + src/domains/memory/distill-trigger.ts | 77 + src/domains/memory/distill-types.test.ts | 209 +++ src/domains/memory/distill-types.ts | 190 +++ src/domains/memory/distill-workflow.test.ts | 48 + src/domains/memory/distill-workflow.ts | 231 +++ src/domains/memory/extract-workflow.test.ts | 40 + src/domains/memory/extract-workflow.ts | 87 +- src/domains/memory/extraction-model.ts | 33 +- src/domains/memory/observation-types.test.ts | 78 + src/domains/memory/observation-types.ts | 50 + src/domains/memory/prompts.test.ts | 33 +- src/domains/memory/prompts.ts | 279 +--- src/domains/memory/repository.ts | 428 +++-- src/domains/memory/resolve-operations.test.ts | 2 +- src/domains/memory/resolve-operations.ts | 36 +- src/domains/memory/service.ts | 66 +- src/infrastructure/db/schema.ts | 50 + 29 files changed, 3562 insertions(+), 486 deletions(-) create mode 100644 .pnpm-store/v11/index.db create mode 160000 .repos/OpenViking create mode 100644 drizzle/0015_bumpy_vulcan.sql create mode 100644 drizzle/meta/0015_snapshot.json create mode 100644 src/app/api/memory/distill/route.ts create mode 100644 src/domains/memory/distill-config.ts create mode 100644 src/domains/memory/distill-model.ts create mode 100644 src/domains/memory/distill-prompts.test.ts create mode 100644 src/domains/memory/distill-prompts.ts create mode 100644 src/domains/memory/distill-trigger.test.ts create mode 100644 src/domains/memory/distill-trigger.ts create mode 100644 src/domains/memory/distill-types.test.ts create mode 100644 src/domains/memory/distill-types.ts create mode 100644 src/domains/memory/distill-workflow.test.ts create mode 100644 src/domains/memory/distill-workflow.ts create mode 100644 src/domains/memory/extract-workflow.test.ts create mode 100644 src/domains/memory/observation-types.test.ts create mode 100644 src/domains/memory/observation-types.ts diff --git a/.pnpm-store/v11/index.db b/.pnpm-store/v11/index.db new file mode 100644 index 0000000000000000000000000000000000000000..8fdf9e7d3d8db312b5d08efe43a80ea9046a773f GIT binary patch literal 8192 zcmeIuKa0XJ7zXe(90-L&2f@*I^A3(%?hDvzz+$vsP3gHPQKLxxhjh@v!PQS~E2W&1 zyF3p`-Xw39{D!}Bl^9y=4jY}&534ZFS(At{#`Bq$d#rSQ%lBEy&dc$*8rJ2U=;_+*^(@v)Gg)ot=J;^ntw1bDd~%rSDSTSJV%sESjGbd8*@wiUL3K k+WEbmsaIZ*9$$h01Rwwb2tWV=5P$##AOHafK%fw~0WYO9761SM literal 0 HcmV?d00001 diff --git a/.repos/OpenViking b/.repos/OpenViking new file mode 160000 index 0000000..f6d9dec --- /dev/null +++ b/.repos/OpenViking @@ -0,0 +1 @@ +Subproject commit f6d9dec6b6ae16a152c437fd4ad81ca45fcc8648 diff --git a/drizzle/0015_bumpy_vulcan.sql b/drizzle/0015_bumpy_vulcan.sql new file mode 100644 index 0000000..2b38375 --- /dev/null +++ b/drizzle/0015_bumpy_vulcan.sql @@ -0,0 +1,17 @@ +CREATE TABLE "fluid_observations" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "source_message_id" uuid, + "signal" text NOT NULL, + "evidence_quote" text NOT NULL, + "subject_hint" text, + "referenced_document_ids" jsonb DEFAULT '[]'::jsonb NOT NULL, + "confidence" double precision NOT NULL, + "status" text DEFAULT 'pending' NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "consumed_at" timestamp with time zone +); +--> statement-breakpoint +ALTER TABLE "fluid_observations" ADD CONSTRAINT "fluid_observations_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "fluid_observations" ADD CONSTRAINT "fluid_observations_source_message_id_chat_messages_id_fk" FOREIGN KEY ("source_message_id") REFERENCES "public"."chat_messages"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "fluid_observations_workspace_status_created_idx" ON "fluid_observations" USING btree ("workspace_id","status","created_at"); \ No newline at end of file diff --git a/drizzle/meta/0015_snapshot.json b/drizzle/meta/0015_snapshot.json new file mode 100644 index 0000000..3b313fd --- /dev/null +++ b/drizzle/meta/0015_snapshot.json @@ -0,0 +1,1444 @@ +{ + "id": "be8deb1f-3ec8-4304-85d0-41403629970c", + "prevId": "72bfb4fe-4048-4dce-a5fa-ce8eea880aa3", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_memory_items": { + "name": "fluid_memory_items", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "abstract_l0": { + "name": "abstract_l0", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "overview_l1": { + "name": "overview_l1", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "fluid_memory_items_workspace_status_idx": { + "name": "fluid_memory_items_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fluid_memory_items_workspace_kind_idx": { + "name": "fluid_memory_items_workspace_kind_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_memory_items_workspace_id_workspaces_id_fk": { + "name": "fluid_memory_items_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_memory_items_source_message_id_chat_messages_id_fk": { + "name": "fluid_memory_items_source_message_id_chat_messages_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_memory_tokens": { + "name": "fluid_memory_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "item_id": { + "name": "item_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "frequency": { + "name": "frequency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + } + }, + "indexes": { + "fluid_memory_tokens_lookup_idx": { + "name": "fluid_memory_tokens_lookup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fluid_memory_tokens_item_idx": { + "name": "fluid_memory_tokens_item_idx", + "columns": [ + { + "expression": "item_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_memory_tokens_workspace_id_workspaces_id_fk": { + "name": "fluid_memory_tokens_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_memory_tokens", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_memory_tokens_item_id_fluid_memory_items_id_fk": { + "name": "fluid_memory_tokens_item_id_fluid_memory_items_id_fk", + "tableFrom": "fluid_memory_tokens", + "tableTo": "fluid_memory_items", + "columnsFrom": [ + "item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_observations": { + "name": "fluid_observations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "signal": { + "name": "signal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "evidence_quote": { + "name": "evidence_quote", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "subject_hint": { + "name": "subject_hint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "referenced_document_ids": { + "name": "referenced_document_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "confidence": { + "name": "confidence", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "fluid_observations_workspace_status_created_idx": { + "name": "fluid_observations_workspace_status_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_observations_workspace_id_workspaces_id_fk": { + "name": "fluid_observations_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_observations", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_observations_source_message_id_chat_messages_id_fk": { + "name": "fluid_observations_source_message_id_chat_messages_id_fk", + "tableFrom": "fluid_observations", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_diffs": { + "name": "memory_diffs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memory_diffs_workspace_created_idx": { + "name": "memory_diffs_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_diffs_workspace_id_workspaces_id_fk": { + "name": "memory_diffs_workspace_id_workspaces_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memory_diffs_source_message_id_chat_messages_id_fk": { + "name": "memory_diffs_source_message_id_chat_messages_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parsed_document_sync_leases": { + "name": "parsed_document_sync_leases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_token": { + "name": "lease_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_reason": { + "name": "release_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parsed_document_sync_leases_token_idx": { + "name": "parsed_document_sync_leases_token_idx", + "columns": [ + { + "expression": "lease_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_active_idx": { + "name": "parsed_document_sync_leases_active_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_workspace_active_idx": { + "name": "parsed_document_sync_leases_workspace_active_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_document_active_idx": { + "name": "parsed_document_sync_leases_document_active_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parsed_document_sync_leases_workspace_id_workspaces_id_fk": { + "name": "parsed_document_sync_leases_workspace_id_workspaces_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parsed_document_sync_leases_source_id_sources_id_fk": { + "name": "parsed_document_sync_leases_source_id_sources_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index b9be4f5..32ea955 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -106,6 +106,13 @@ "when": 1788252035435, "tag": "0014_messy_apocalypse", "breakpoints": true + }, + { + "idx": 15, + "version": "7", + "when": 1788418416499, + "tag": "0015_bumpy_vulcan", + "breakpoints": true } ] } \ No newline at end of file diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 80ee5bb..082ea2d 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,3 +1,11 @@ +allowBuilds: + core-js: set this to true or false + esbuild: set this to true or false + msgpackr-extract: set this to true or false + msw: set this to true or false + sharp: set this to true or false + unrs-resolver: set this to true or false + ignoredBuiltDependencies: - sharp - unrs-resolver diff --git a/src/app/api/memory/distill/route.ts b/src/app/api/memory/distill/route.ts new file mode 100644 index 0000000..e0230d9 --- /dev/null +++ b/src/app/api/memory/distill/route.ts @@ -0,0 +1,27 @@ +import { serve } from "@upstash/workflow/nextjs" + +import { + normalizeMemoryDistillPayload, + runMemoryDistillWorkflow, + type MemoryDistillPayload, +} from "@/domains/memory/distill-workflow" +import { logger } from "@/lib/logger" + +export const { POST } = serve( + async (context) => { + const payload = normalizeMemoryDistillPayload(context.requestPayload) + if (!payload) { + logger.warn("memory: distill workflow received invalid payload") + return + } + await runMemoryDistillWorkflow({ context, payload }) + }, + { + failureFunction: async ({ context, failResponse }) => { + logger.error("memory: distill workflow failed", { + payload: context.requestPayload, + failResponse, + }) + }, + }, +) diff --git a/src/domains/memory/distill-config.ts b/src/domains/memory/distill-config.ts new file mode 100644 index 0000000..4c82abb --- /dev/null +++ b/src/domains/memory/distill-config.ts @@ -0,0 +1,19 @@ +/** + * Distill job defaults from the two-tier fluid-memory plan. + * Tunable later; kept as named constants (not scattered literals). + */ + +/** Process-local cooldown + QStash workflowRunId bucket width (mirrors reconcile). */ +export const DISTILL_COOLDOWN_MS = 5 * 60_000 + +/** Capture may trigger distill once pending observations reach this count. */ +export const DISTILL_MIN_PENDING = 8 + +/** Max pending rows claimed per distill run (oldest first). */ +export const DISTILL_BATCH_MAX = 40 + +/** Lexical dedup candidates loaded per memory kind for one distill batch. */ +export const DISTILL_DEDUP_CANDIDATES_PER_KIND = 8 + +/** Delete consumed observations older than this (retention sweep after distill). */ +export const DISTILL_CONSUMED_RETENTION_MS = 30 * 24 * 60_000 diff --git a/src/domains/memory/distill-model.ts b/src/domains/memory/distill-model.ts new file mode 100644 index 0000000..ef2485f --- /dev/null +++ b/src/domains/memory/distill-model.ts @@ -0,0 +1,83 @@ +import "server-only" + +import { generateObject } from "ai" + +import { buildDistillPrompt } from "./distill-prompts" +import { + entityDistillOutputSchema, + experienceDistillOutputSchema, + indicatorDistillOutputSchema, + toMemoryOperations, + type DistillObservationInput, + type DistillPassKind, + type ExistingMemoryContextItem, + type MemoryOperations, +} from "./distill-types" +import { CHAT_MODEL } from "@/lib/ai" +import { summarizeUnknownError } from "@/lib/format-log-value" +import { logger } from "@/lib/logger" + +const MEMORY_EXTRACTION_MODEL = process.env.MEMORY_EXTRACTION_MODEL ?? CHAT_MODEL + +/** + * One structured-output call for a single distill pass. + * Best-effort — distill runs as a background job; a model failure returns + * null so the workflow can skip applying that pass (logged). No multi-level + * fallback chain. + * + * Input: pending observation batch + existing memories of kinds this pass may + * write + allowed document ids for the batch. + * Output: full MemoryOperations with only this pass's arrays populated + * (others empty), ready for resolveMemoryOperations. + */ +export async function distillMemoryPass(input: { + readonly pass: DistillPassKind + readonly workspaceId: string + readonly observations: readonly DistillObservationInput[] + readonly existingItems: readonly ExistingMemoryContextItem[] + readonly referencedDocumentIds: readonly string[] +}): Promise { + const prompt = buildDistillPrompt(input.pass, { + observations: input.observations, + existingItems: input.existingItems, + referencedDocumentIds: input.referencedDocumentIds, + }) + + try { + switch (input.pass) { + case "indicator": { + const response = await generateObject({ + model: MEMORY_EXTRACTION_MODEL, + schema: indicatorDistillOutputSchema, + messages: [{ role: "user", content: prompt }], + }) + return toMemoryOperations("indicator", response.object) + } + case "experience": { + const response = await generateObject({ + model: MEMORY_EXTRACTION_MODEL, + schema: experienceDistillOutputSchema, + messages: [{ role: "user", content: prompt }], + }) + return toMemoryOperations("experience", response.object) + } + case "entity": { + const response = await generateObject({ + model: MEMORY_EXTRACTION_MODEL, + schema: entityDistillOutputSchema, + messages: [{ role: "user", content: prompt }], + }) + return toMemoryOperations("entity", response.object) + } + } + } catch (error) { + logger.warn("memory: distill model call failed; skipping pass", { + workspaceId: input.workspaceId, + pass: input.pass, + model: MEMORY_EXTRACTION_MODEL, + observationCount: input.observations.length, + error: summarizeUnknownError(error), + }) + return null + } +} diff --git a/src/domains/memory/distill-prompts.test.ts b/src/domains/memory/distill-prompts.test.ts new file mode 100644 index 0000000..9457880 --- /dev/null +++ b/src/domains/memory/distill-prompts.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from "vitest" + +import { buildDistillPrompt } from "./distill-prompts" +import type { DistillObservationInput } from "./distill-types" + +const observations: DistillObservationInput[] = [ + { + id: "obs-1", + signal: "看重毛利率", + evidenceQuote: "毛利率是核心", + subjectHint: "毛利率", + confidence: 0.9, + referencedDocumentIds: ["doc-1"], + }, + { + id: "obs-2", + signal: "跟踪英伟达", + evidenceQuote: "英伟达一直在跟踪", + subjectHint: "英伟达", + confidence: 0.8, + referencedDocumentIds: [], + }, +] + +describe("buildDistillPrompt", () => { + it("indicator pass: only indicator schema, no other kind arrays", () => { + const prompt = buildDistillPrompt("indicator", { + observations, + existingItems: [ + { + id: "item-1", + kind: "indicator_pref", + abstractL0: "看重毛利率", + payloadSummary: "毛利率 — 毛利占营收", + }, + ], + referencedDocumentIds: ["doc-1"], + }) + + expect(prompt).toContain("You DISTILL indicator preferences") + expect(prompt).toContain('"indicatorPrefs"') + expect(prompt).not.toContain('"stances"') + expect(prompt).not.toContain('"decisionRules"') + expect(prompt).not.toContain('"entities"') + expect(prompt).toContain("never emit") + expect(prompt).toContain("PENDING OBSERVATIONS") + expect(prompt).toContain("id=obs-1") + expect(prompt).toContain("id=obs-2") + expect(prompt).toContain("看重毛利率") + expect(prompt).toContain("跟踪英伟达") + expect(prompt).toContain("id=item-1") + expect(prompt).toContain("doc-1") + // All observations go to every pass — no kind routing. + expect(prompt.indexOf("obs-1")).toBeLessThan(prompt.indexOf("obs-2")) + }) + + it("experience pass: stances+rules only; still sees full observation batch", () => { + const prompt = buildDistillPrompt("experience", { + observations, + existingItems: [], + referencedDocumentIds: [], + }) + + expect(prompt).toContain("You DISTILL stances and decision rules") + expect(prompt).toContain('"stances"') + expect(prompt).toContain('"decisionRules"') + expect(prompt).not.toContain('"indicatorPrefs"') + expect(prompt).not.toContain('"entities"') + expect(prompt).toContain("id=obs-1") + expect(prompt).toContain("id=obs-2") + expect(prompt).toContain("(no existing memories yet)") + expect(prompt).toContain("(no documents referenced in this batch)") + }) + + it("entity pass: entities only; referenced ids from batch", () => { + const prompt = buildDistillPrompt("entity", { + observations, + existingItems: [ + { + id: "item-4", + kind: "entity_of_interest", + abstractL0: "跟踪英伟达", + payloadSummary: "英伟达 NVDA", + }, + ], + referencedDocumentIds: ["doc-1"], + }) + + expect(prompt).toContain("You DISTILL entities of interest") + expect(prompt).toContain('"entities"') + expect(prompt).not.toContain('"indicatorPrefs"') + expect(prompt).not.toContain('"stances"') + expect(prompt).not.toContain('"decisionRules"') + expect(prompt).toContain("never invent ids") + expect(prompt).toContain("id=item-4") + expect(prompt).toContain("doc-1") + }) + + it("keeps illustrative examples separated from main instructions", () => { + const prompt = buildDistillPrompt("indicator", { + observations: [], + existingItems: [], + referencedDocumentIds: [], + }) + const main = prompt.slice(0, prompt.indexOf("## Illustrative examples")) + expect(main).not.toMatch(/gross margin|市盈率|\bPE\b|Fed|美联储|NVIDIA|英伟达/i) + expect(prompt).toContain("## Illustrative examples (finance vertical") + expect(prompt).toContain("(no pending observations)") + }) +}) diff --git a/src/domains/memory/distill-prompts.ts b/src/domains/memory/distill-prompts.ts new file mode 100644 index 0000000..3e9e70c --- /dev/null +++ b/src/domains/memory/distill-prompts.ts @@ -0,0 +1,311 @@ +import type { + DistillObservationInput, + DistillPassKind, + ExistingMemoryContextItem, +} from "./distill-types" + +/** + * Distill prompts — three isolated passes over the same pending observation + * batch. Each pass sees ALL observations (no kind routing) and only the + * existing memories of kinds that pass may write. + */ + +const INDICATOR_OUTPUT_SCHEMA_BLOCK = `{ + "indicatorPrefs": [{ + "name": "string", + "aliases": ["string"], + "definition": "string", + "polarity": "higher_better|lower_better|context", + "importance": "core|secondary", + "formulaHint": "string (optional — omit if none)", + "abstractL0": "string", + "overviewL1": "string", + "confidence": 0.0, + "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } + }] +}` + +const EXPERIENCE_OUTPUT_SCHEMA_BLOCK = `{ + "stances": [{ + "statement": "string (the stance text; do not use a name field)", + "scope": "string", + "rationale": "string", + "abstractL0": "string", + "overviewL1": "string", + "confidence": 0.0, + "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } + }], + "decisionRules": [{ + "when": "string", + "then": "string", + "priority": "high|medium|low", + "rationale": "string", + "abstractL0": "string", + "overviewL1": "string", + "confidence": 0.0, + "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } + }] +}` + +const ENTITY_OUTPUT_SCHEMA_BLOCK = `{ + "entities": [{ + "name": "string", + "ticker": "string optional", + "aliases": ["string"], + "knowhereDocumentIds": ["only ids listed under REFERENCED DOCUMENT IDS"], + "reason": "string", + "abstractL0": "string", + "overviewL1": "string", + "confidence": 0.0, + "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } + }] +}` + +const INDICATOR_ILLUSTRATIVE_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) + +These show shape and judgement only. Distill whatever the observations support; +do not force the batch into this domain or these metric names. + +- Recurring evaluation metric named across clues → one indicatorPref (stable name + definition + polarity + importance). +- Same metric restated with a nuance → merge into the existing item, do not create a second. +- Skip: a one-off number question, document fact, or weak single-mention with no reusable criterion.` + +const EXPERIENCE_ILLUSTRATIVE_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) + +These show shape and judgement only. Distill whatever the observations support; +do not force the batch into this domain. + +- Durable judgement frame (e.g. long-horizon) → one stance (statement + scope + rationale). +- Reusable when → then discipline over the user's criteria → one decisionRule. +- Abstract away one-off instances; keep a single intent per rule. Split unrelated intents. +- Skip: process narration, document facts, or a preference that is only a metric definition (indicators are another pass).` + +const ENTITY_ILLUSTRATIVE_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) + +These show shape and judgement only. Distill whatever the observations support; +do not force the batch into this domain. + +- User actively tracks a named company/issuer across clues → one entity (name + reason; optional ticker/aliases). +- Same subject restated → merge; attach knowhereDocumentIds only from REFERENCED DOCUMENT IDS. +- Skip: a company mentioned only as a one-off fact question, or names that are not subjects of ongoing interest.` + +const INDICATOR_INSTRUCTIONS_BLOCK = `You DISTILL indicator preferences for a user's fluid memory. + +You receive a BATCH of raw observations (cheap clues about what the USER cares +about) plus existing indicator memories. Produce durable indicator_pref items +only. A separate pass handles stances, decision rules, and entities — never emit +those kinds here. + +Constraints: +- One stable topic/name per preference; merge overlapping or synonymous names. +- Capture "what the user repeatedly uses to evaluate", not one-off facts. +- Keep unrelated criteria as separate items; do not mix them into one payload. + +## What to emit + +indicatorPrefs — recurring metrics or criteria the user uses to evaluate things. +Fields: name, aliases, definition, polarity (higher_better | lower_better | context), +importance (core | secondary), optional formulaHint, abstractL0, overviewL1, +confidence, decision. + +## Language + +- Keep this instruction set and enum/field names in English. +- Write every free-text value in the same language the observations use. Do not + translate the user's terms into English unless the observations themselves used English. + +## Decision rules + +- Distill only from the observation batch evidence about the USER. +- Skip document facts, retrieved numbers, and weak/ephemeral clues. +- For every candidate, choose exactly one op against EXISTING MEMORIES: + - create — genuinely new indicator + - skip — already covered, or too weak + - merge — same indicator refined; emit the full merged fields and set targetItemId + - deprecate — user clearly reversed a stored indicator; set targetItemId +- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. +- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when explicitly stated. +- Omit optional fields instead of setting them to null. +- If nothing qualifies, return {"indicatorPrefs": []}.` + +const EXPERIENCE_INSTRUCTIONS_BLOCK = `You DISTILL stances and decision rules (insights) for a user's fluid memory. + +You receive a BATCH of raw observations plus existing stance/decision-rule +memories. Produce durable stances and decisionRules only. A separate pass +handles indicators and entities — never emit those kinds here. + +Constraints: +- Generalizable, reusable insight — not a process log of one session. +- Atomic scope: one intent per decisionRule; split if when would mix goals. +- Abstract away specific one-off entities/ids from the situation framing when the + rule itself is general; keep concrete names only when the insight requires them. +- Do not restate a bare metric definition as a decisionRule — that belongs to the indicator pass. + +## What to emit + +- stances — durable positions that shape how the user weighs evidence. + Fields: statement (required; do not invent a "name" field), scope, rationale, + abstractL0, overviewL1, confidence, decision. +- decisionRules — reusable when → then disciplines the user stated or clearly endorsed. + Fields: when, then, priority (high | medium | low), rationale, abstractL0, + overviewL1, confidence, decision. + +## Language + +- Keep this instruction set and enum/field names in English. +- Write every free-text value in the same language the observations use. Do not + translate the user's terms into English unless the observations themselves used English. + +## Decision rules + +- Distill only from the observation batch evidence about the USER. +- Skip document facts, small talk, and weak/ephemeral clues. +- For every candidate, choose exactly one op against EXISTING MEMORIES: + - create — genuinely new + - skip — already covered, or too weak + - merge — same insight refined; emit the full merged fields and set targetItemId + - deprecate — user clearly reversed a stored item; set targetItemId +- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. +- Prefer one record per insight. Do not invent a near-duplicate decisionRule for a stance that already encodes the same frame unless the user stated an explicit when → then action. +- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when explicitly stated. +- Omit optional fields instead of setting them to null. +- If nothing qualifies, return {"stances": [], "decisionRules": []}.` + +const ENTITY_INSTRUCTIONS_BLOCK = `You DISTILL entities of interest for a user's fluid memory. + +You receive a BATCH of raw observations plus existing entity memories. Produce +durable entity_of_interest items only. A separate pass handles indicators, +stances, and decision rules — never emit those kinds here. + +Constraints: +- Stable card for a subject the USER actively tracks. +- Merge overlapping names/aliases into one item; keep unrelated subjects separate. +- Attach document provenance only from ids listed under REFERENCED DOCUMENT IDS. + +## What to emit + +entities — named subjects the user is actively tracking. +Fields: name, optional ticker, aliases, reason (required), knowhereDocumentIds +(only from REFERENCED DOCUMENT IDS below; never invent ids), abstractL0, +overviewL1, confidence, decision. + +## Language + +- Keep this instruction set and enum/field names in English. +- Write every free-text value in the same language the observations use. Do not + translate the user's terms into English unless the observations themselves used English. + +## Decision rules + +- Distill only from the observation batch evidence about the USER. +- Skip one-off name drops, document facts, and weak/ephemeral mentions. +- For every candidate, choose exactly one op against EXISTING MEMORIES: + - create — genuinely new tracked subject + - skip — already covered, or too weak + - merge — same subject refined; emit the full merged fields and set targetItemId + - deprecate — user clearly stopped tracking / reversed; set targetItemId +- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. +- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when explicitly stated. +- Omit optional fields instead of setting them to null. +- If nothing qualifies, return {"entities": []}.` + +export type BuildDistillPromptInput = { + readonly observations: readonly DistillObservationInput[] + readonly existingItems: readonly ExistingMemoryContextItem[] + readonly referencedDocumentIds: readonly string[] +} + +export function buildDistillPrompt( + pass: DistillPassKind, + input: BuildDistillPromptInput, +): string { + switch (pass) { + case "indicator": + return assemblePrompt({ + instructions: INDICATOR_INSTRUCTIONS_BLOCK, + examples: INDICATOR_ILLUSTRATIVE_BLOCK, + outputSchema: INDICATOR_OUTPUT_SCHEMA_BLOCK, + input, + }) + case "experience": + return assemblePrompt({ + instructions: EXPERIENCE_INSTRUCTIONS_BLOCK, + examples: EXPERIENCE_ILLUSTRATIVE_BLOCK, + outputSchema: EXPERIENCE_OUTPUT_SCHEMA_BLOCK, + input, + }) + case "entity": + return assemblePrompt({ + instructions: ENTITY_INSTRUCTIONS_BLOCK, + examples: ENTITY_ILLUSTRATIVE_BLOCK, + outputSchema: ENTITY_OUTPUT_SCHEMA_BLOCK, + input, + }) + } +} + +function assemblePrompt(args: { + readonly instructions: string + readonly examples: string + readonly outputSchema: string + readonly input: BuildDistillPromptInput +}): string { + const existingBlock = + args.input.existingItems.length === 0 + ? "(no existing memories yet)" + : args.input.existingItems + .map( + (item) => + `- [${item.kind}] id=${item.id} :: ${item.abstractL0} :: ${item.payloadSummary}`, + ) + .join("\n") + + const documentsBlock = + args.input.referencedDocumentIds.length === 0 + ? "(no documents referenced in this batch)" + : args.input.referencedDocumentIds.join(", ") + + const observationsBlock = + args.input.observations.length === 0 + ? "(no pending observations)" + : args.input.observations + .map((observation) => formatObservation(observation)) + .join("\n\n") + + return `${args.instructions} + +${args.examples} + +## Output JSON schema (follow exactly; do not invent fields) + +${args.outputSchema} + +## EXISTING MEMORIES + +${existingBlock} + +## REFERENCED DOCUMENT IDS + +${documentsBlock} + +## PENDING OBSERVATIONS + +${observationsBlock}` +} + +function formatObservation(observation: DistillObservationInput): string { + const subject = + observation.subjectHint && observation.subjectHint.length > 0 + ? observation.subjectHint + : "(none)" + const docs = + observation.referencedDocumentIds.length === 0 + ? "(none)" + : observation.referencedDocumentIds.join(", ") + return `- id=${observation.id} + signal: ${observation.signal} + evidenceQuote: ${observation.evidenceQuote} + subjectHint: ${subject} + confidence: ${observation.confidence} + referencedDocumentIds: ${docs}` +} diff --git a/src/domains/memory/distill-trigger.test.ts b/src/domains/memory/distill-trigger.test.ts new file mode 100644 index 0000000..f4a09d1 --- /dev/null +++ b/src/domains/memory/distill-trigger.test.ts @@ -0,0 +1,84 @@ +import { afterEach, describe, expect, it, vi } from "vitest" + +import { DISTILL_COOLDOWN_MS, DISTILL_MIN_PENDING } from "./distill-config" + +const mocks = vi.hoisted(() => ({ + loggerError: vi.fn(), + loggerInfo: vi.fn(), + loggerWarn: vi.fn(), + trigger: vi.fn(), + countPendingObservations: vi.fn(), +})) + +vi.mock("@upstash/workflow", () => ({ + Client: class { + trigger = mocks.trigger + }, +})) + +vi.mock("@/lib/logger", () => ({ + logger: { + error: mocks.loggerError, + info: mocks.loggerInfo, + warn: mocks.loggerWarn, + }, +})) + +vi.mock("./service", () => ({ + memoryService: { + countPendingObservations: mocks.countPendingObservations, + }, +})) + +describe("triggerMemoryDistill", () => { + afterEach(async () => { + vi.clearAllMocks() + vi.useRealTimers() + delete process.env.QSTASH_TOKEN + delete process.env.NOTEBOOK_PUBLIC_URL + const { resetMemoryDistillTriggerStateForTests } = await import( + "./distill-trigger" + ) + resetMemoryDistillTriggerStateForTests() + vi.resetModules() + }) + + it("does not trigger when pending is below the plan threshold", async () => { + mocks.countPendingObservations.mockResolvedValue(DISTILL_MIN_PENDING - 1) + process.env.QSTASH_TOKEN = "qstash_token" + + const { triggerMemoryDistill } = await import("./distill-trigger") + await triggerMemoryDistill({ workspaceId: "workspace_1" }) + + expect(mocks.trigger).not.toHaveBeenCalled() + }) + + it("deduplicates workflow triggers only within a bounded cooldown", async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date("2026-06-30T00:00:00.000Z")) + process.env.QSTASH_TOKEN = "qstash_token" + process.env.NOTEBOOK_PUBLIC_URL = "https://notebook.example" + mocks.countPendingObservations.mockResolvedValue(DISTILL_MIN_PENDING) + mocks.trigger.mockResolvedValue({}) + + const { triggerMemoryDistill } = await import("./distill-trigger") + + await triggerMemoryDistill({ workspaceId: "workspace_1" }) + await triggerMemoryDistill({ workspaceId: "workspace_1" }) + + expect(mocks.trigger).toHaveBeenCalledTimes(1) + expect(mocks.trigger).toHaveBeenLastCalledWith({ + url: "https://notebook.example/api/memory/distill", + body: { workspaceId: "workspace_1" }, + workflowRunId: `workspace_1-${Math.floor( + new Date("2026-06-30T00:00:00.000Z").getTime() / DISTILL_COOLDOWN_MS, + )}`, + retries: 3, + }) + + vi.setSystemTime(new Date("2026-06-30T00:05:01.000Z")) + await triggerMemoryDistill({ workspaceId: "workspace_1" }) + + expect(mocks.trigger).toHaveBeenCalledTimes(2) + }) +}) diff --git a/src/domains/memory/distill-trigger.ts b/src/domains/memory/distill-trigger.ts new file mode 100644 index 0000000..a9bd0fd --- /dev/null +++ b/src/domains/memory/distill-trigger.ts @@ -0,0 +1,77 @@ +import "server-only" + +import { Client } from "@upstash/workflow" + +import { DISTILL_COOLDOWN_MS, DISTILL_MIN_PENDING } from "./distill-config" +import type { MemoryDistillPayload } from "./distill-workflow" +import { memoryService } from "./service" +import { logger } from "@/lib/logger" + +// Re-trigger protection: process-local cooldown + bucketed workflowRunId. +// Mirrors background-reconcile — same cooldown width keys both guards. + +const lastTriggeredAtByWorkspaceId: Map = new Map() + +function resolveBaseURL(): string { + return process.env.NOTEBOOK_PUBLIC_URL ?? "http://localhost:3000" +} + +/** + * Fire-and-forget distill trigger for one workspace. + * Caller should already know pending may be high; this re-checks the count, + * applies cooldown + bucketed workflowRunId, then enqueues QStash. + */ +export async function triggerMemoryDistill( + payload: MemoryDistillPayload, +): Promise { + const pendingCount = await memoryService.countPendingObservations( + payload.workspaceId, + ) + if (pendingCount < DISTILL_MIN_PENDING) return + + const now = Date.now() + const lastTriggeredAt = lastTriggeredAtByWorkspaceId.get(payload.workspaceId) + if ( + lastTriggeredAt !== undefined && + now - lastTriggeredAt < DISTILL_COOLDOWN_MS + ) { + return + } + lastTriggeredAtByWorkspaceId.set(payload.workspaceId, now) + + const token = process.env.QSTASH_TOKEN + if (!token) { + logger.warn("memory: skipping distill — QSTASH_TOKEN not set", { + workspaceId: payload.workspaceId, + pendingCount, + }) + lastTriggeredAtByWorkspaceId.delete(payload.workspaceId) + return + } + + const url = `${resolveBaseURL()}/api/memory/distill` + try { + await new Client({ token }).trigger({ + url, + body: payload, + workflowRunId: `${payload.workspaceId}-${Math.floor(now / DISTILL_COOLDOWN_MS)}`, + retries: 3, + }) + logger.info("memory: distill workflow triggered", { + workspaceId: payload.workspaceId, + pendingCount, + url, + }) + } catch (error) { + lastTriggeredAtByWorkspaceId.delete(payload.workspaceId) + logger.error("memory: failed to trigger distill workflow", { + workspaceId: payload.workspaceId, + message: error instanceof Error ? error.message : String(error), + }) + } +} + +/** Test helper: clear process-local cooldown map between cases. */ +export function resetMemoryDistillTriggerStateForTests(): void { + lastTriggeredAtByWorkspaceId.clear() +} diff --git a/src/domains/memory/distill-types.test.ts b/src/domains/memory/distill-types.test.ts new file mode 100644 index 0000000..b3b161d --- /dev/null +++ b/src/domains/memory/distill-types.test.ts @@ -0,0 +1,209 @@ +import { describe, expect, it } from "vitest" + +import { + entityDistillOutputSchema, + experienceDistillOutputSchema, + indicatorDistillOutputSchema, + memoryOperationsSchema, + summarizePayloadForContext, + toMemoryOperations, +} from "./distill-types" + +describe("distill output schemas", () => { + it("indicator pass accepts prefs and defaults empty array", () => { + expect(indicatorDistillOutputSchema.parse({})).toEqual({ + indicatorPrefs: [], + }) + const parsed = indicatorDistillOutputSchema.parse({ + indicatorPrefs: [ + { + name: "毛利率", + aliases: [], + definition: "毛利占营收", + polarity: "higher_better", + importance: "core", + abstractL0: "看重毛利率", + overviewL1: "用户用毛利率判断质量。", + confidence: 0.9, + decision: { op: "create" }, + }, + ], + }) + expect(parsed.indicatorPrefs).toHaveLength(1) + expect(parsed).not.toHaveProperty("stances") + }) + + it("experience pass accepts stance+rule when required fields are present", () => { + const parsed = experienceDistillOutputSchema.parse({ + stances: [ + { + statement: "长期持有", + scope: "投资 horizon", + rationale: "用户明确说长期", + abstractL0: "长期持有", + overviewL1: "用户以长期视角评估。", + confidence: 1, + decision: { op: "create" }, + }, + ], + decisionRules: [ + { + when: "毛利率连续两季下滑", + then: "减仓观望", + priority: "high", + rationale: "用户自述纪律", + abstractL0: "毛利率下滑则减仓", + overviewL1: "连续两季下滑时减仓观望。", + confidence: 0.95, + decision: { op: "create" }, + }, + ], + }) + expect(parsed.stances[0]?.statement).toBe("长期持有") + expect(parsed.decisionRules).toHaveLength(1) + expect(parsed).not.toHaveProperty("indicatorPrefs") + }) + + it("rejects stance that uses name instead of statement (no coerce)", () => { + expect(() => + experienceDistillOutputSchema.parse({ + stances: [ + { + name: "长期持有", + scope: "投资", + rationale: "用户明确说长期", + abstractL0: "长期持有", + overviewL1: "用户以长期视角评估。", + confidence: 1, + decision: { op: "create" }, + }, + ], + }), + ).toThrow() + }) + + it("rejects entity missing reason (no fill from abstractL0)", () => { + expect(() => + entityDistillOutputSchema.parse({ + entities: [ + { + name: "英伟达", + aliases: ["NVIDIA"], + knowhereDocumentIds: ["doc-1"], + abstractL0: "持续跟踪英伟达", + overviewL1: "用户把英伟达列为跟踪标的。", + confidence: 0.8, + decision: { op: "merge", targetItemId: "item-4" }, + }, + ], + }), + ).toThrow() + }) + + it("entity pass accepts when reason is present", () => { + const parsed = entityDistillOutputSchema.parse({ + entities: [ + { + name: "英伟达", + aliases: ["NVIDIA"], + knowhereDocumentIds: ["doc-1"], + reason: "用户持续跟踪", + abstractL0: "持续跟踪英伟达", + overviewL1: "用户把英伟达列为跟踪标的。", + confidence: 0.8, + decision: { op: "merge", targetItemId: "item-4" }, + }, + ], + }) + expect(parsed.entities[0]?.reason).toBe("用户持续跟踪") + }) + + it("toMemoryOperations expands each pass into the full four-array record", () => { + expect( + toMemoryOperations("indicator", { + indicatorPrefs: [], + }), + ).toEqual({ + indicatorPrefs: [], + stances: [], + decisionRules: [], + entities: [], + }) + + const experience = toMemoryOperations( + "experience", + experienceDistillOutputSchema.parse({ + stances: [ + { + statement: "长期", + scope: "投资", + rationale: "用户说的", + abstractL0: "长期", + overviewL1: "长期视角。", + confidence: 1, + decision: { op: "create" }, + }, + ], + }), + ) + expect(experience.stances).toHaveLength(1) + expect(experience.indicatorPrefs).toEqual([]) + expect(experience.entities).toEqual([]) + + const entity = toMemoryOperations( + "entity", + entityDistillOutputSchema.parse({ + entities: [ + { + name: "英伟达", + aliases: [], + knowhereDocumentIds: [], + reason: "跟踪", + abstractL0: "跟踪英伟达", + overviewL1: "用户跟踪英伟达。", + confidence: 0.7, + decision: { op: "create" }, + }, + ], + }), + ) + expect(entity.entities).toHaveLength(1) + expect(entity.decisionRules).toEqual([]) + }) + + it("memoryOperationsSchema parses the full four-array shape", () => { + const parsed = memoryOperationsSchema.parse({}) + expect(parsed).toEqual({ + indicatorPrefs: [], + stances: [], + decisionRules: [], + entities: [], + }) + }) +}) + +describe("summarizePayloadForContext", () => { + it("summarizes each kind for existing-memory context lines", () => { + expect( + summarizePayloadForContext("indicator_pref", { + name: "毛利率", + definition: "毛利占营收", + }), + ).toBe("毛利率 — 毛利占营收") + expect( + summarizePayloadForContext("stance", { statement: "长期持有" }), + ).toBe("长期持有") + expect( + summarizePayloadForContext("decision_rule", { + when: "下滑", + then: "减仓", + }), + ).toBe("下滑 => 减仓") + expect( + summarizePayloadForContext("entity_of_interest", { + name: "英伟达", + ticker: "NVDA", + }), + ).toBe("英伟达 NVDA") + }) +}) diff --git a/src/domains/memory/distill-types.ts b/src/domains/memory/distill-types.ts new file mode 100644 index 0000000..44b2274 --- /dev/null +++ b/src/domains/memory/distill-types.ts @@ -0,0 +1,190 @@ +import { z } from "zod" + +import { + decisionRulePayloadSchema, + entityOfInterestPayloadSchema, + indicatorPreferencePayloadSchema, + stancePayloadSchema, + type FluidMemoryKind, +} from "./types" + +/** + * Distill-pass LLM contracts. + * + * Three separate structured-output schemas (indicator / experience / + * entity). Capture never emits these shapes — distill is the only writer + * of create/merge/deprecate decisions over `fluid_memory_items`. + */ + +function nullToUndefined(value: unknown): unknown { + return value === null ? undefined : value +} + +const decisionSchema = z.object({ + op: z.enum(["create", "skip", "merge", "deprecate"]), + targetItemId: z.preprocess( + nullToUndefined, + z + .string() + .optional() + .describe( + "Required for merge/deprecate: id of the existing memory item. Omit for create/skip.", + ), + ), + reason: z.preprocess( + nullToUndefined, + z + .string() + .optional() + .describe("Short justification, especially for skip/merge/deprecate."), + ), +}) + +const memorySidecarFields = { + abstractL0: z + .string() + .min(1) + .describe("One line, <= 30 words: the essence of this insight."), + overviewL1: z + .string() + .min(1) + .describe("2-3 sentences: what it means and when it applies."), + confidence: z + .number() + .min(0) + .max(1) + .describe("How explicitly the user stated this across the batch (1 = explicit)."), + decision: decisionSchema, +} + +const stanceEntrySchema = stancePayloadSchema.extend(memorySidecarFields) + +const entityEntrySchema = + entityOfInterestPayloadSchema.extend(memorySidecarFields) + +const indicatorEntrySchema = + indicatorPreferencePayloadSchema.extend(memorySidecarFields) + +const decisionRuleEntrySchema = + decisionRulePayloadSchema.extend(memorySidecarFields) + +/** Full four-array shape consumed by resolveMemoryOperations. */ +export const memoryOperationsSchema = z.object({ + indicatorPrefs: z.array(indicatorEntrySchema).default([]), + stances: z.array(stanceEntrySchema).default([]), + decisionRules: z.array(decisionRuleEntrySchema).default([]), + entities: z.array(entityEntrySchema).default([]), +}) + +export type MemoryOperations = z.infer + +/** Pass 1 — indicator preferences only. */ +export const indicatorDistillOutputSchema = z.object({ + indicatorPrefs: z.array(indicatorEntrySchema).default([]), +}) + +/** Pass 2 — stances + decision rules. */ +export const experienceDistillOutputSchema = z.object({ + stances: z.array(stanceEntrySchema).default([]), + decisionRules: z.array(decisionRuleEntrySchema).default([]), +}) + +/** Pass 3 — entities of interest only. */ +export const entityDistillOutputSchema = z.object({ + entities: z.array(entityEntrySchema).default([]), +}) + +export type IndicatorDistillOutput = z.infer +export type ExperienceDistillOutput = z.infer< + typeof experienceDistillOutputSchema +> +export type EntityDistillOutput = z.infer + +export const distillPassKinds = [ + "indicator", + "experience", + "entity", +] as const + +export type DistillPassKind = (typeof distillPassKinds)[number] + +/** Pending observation row shape fed into distill prompts (batch evidence). */ +export type DistillObservationInput = { + readonly id: string + readonly signal: string + readonly evidenceQuote: string + readonly subjectHint: string | null + readonly confidence: number + readonly referencedDocumentIds: readonly string[] +} + +export type ExistingMemoryContextItem = { + readonly id: string + readonly kind: FluidMemoryKind + readonly abstractL0: string + readonly payloadSummary: string +} + +/** Expand a single-pass LLM object into the full MemoryOperations record. */ +export function toMemoryOperations( + pass: DistillPassKind, + output: + | IndicatorDistillOutput + | ExperienceDistillOutput + | EntityDistillOutput, +): MemoryOperations { + switch (pass) { + case "indicator": { + const typed = output as IndicatorDistillOutput + return { + indicatorPrefs: typed.indicatorPrefs, + stances: [], + decisionRules: [], + entities: [], + } + } + case "experience": { + const typed = output as ExperienceDistillOutput + return { + indicatorPrefs: [], + stances: typed.stances, + decisionRules: typed.decisionRules, + entities: [], + } + } + case "entity": { + const typed = output as EntityDistillOutput + return { + indicatorPrefs: [], + stances: [], + decisionRules: [], + entities: typed.entities, + } + } + } +} + +/** Compact payload label for existing-item context in distill prompts. */ +export function summarizePayloadForContext( + kind: FluidMemoryKind, + payload: unknown, +): string { + if (!payload || typeof payload !== "object") return "" + const record = payload as Record + switch (kind) { + case "indicator_pref": + return [record.name, record.definition] + .filter((part) => typeof part === "string" && part.length > 0) + .join(" — ") + case "stance": + return typeof record.statement === "string" ? record.statement : "" + case "decision_rule": + return [record.when, record.then] + .filter((part) => typeof part === "string" && part.length > 0) + .join(" => ") + case "entity_of_interest": + return [record.name, record.ticker] + .filter((part) => typeof part === "string" && part.length > 0) + .join(" ") + } +} diff --git a/src/domains/memory/distill-workflow.test.ts b/src/domains/memory/distill-workflow.test.ts new file mode 100644 index 0000000..a96b576 --- /dev/null +++ b/src/domains/memory/distill-workflow.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest" + +import { + normalizeMemoryDistillPayload, + toDistillObservationInput, +} from "./distill-workflow" +import type { FluidObservation } from "@/infrastructure/db/schema" + +describe("normalizeMemoryDistillPayload", () => { + it("accepts a workspace id", () => { + expect(normalizeMemoryDistillPayload({ workspaceId: "ws-1" })).toEqual({ + workspaceId: "ws-1", + }) + }) + + it("rejects missing or blank workspace id", () => { + expect(normalizeMemoryDistillPayload(null)).toBeNull() + expect(normalizeMemoryDistillPayload({})).toBeNull() + expect(normalizeMemoryDistillPayload({ workspaceId: " " })).toBeNull() + }) +}) + +describe("toDistillObservationInput", () => { + it("maps a pending row without inventing fields", () => { + const row = { + id: "obs-1", + workspaceId: "ws-1", + sourceMessageId: "msg-1", + signal: "看重毛利率", + evidenceQuote: "毛利率是核心", + subjectHint: "毛利率", + referencedDocumentIds: ["doc-1", ""], + confidence: 0.9, + status: "pending", + createdAt: new Date("2026-06-30T00:00:00.000Z"), + consumedAt: null, + } as FluidObservation + + expect(toDistillObservationInput(row)).toEqual({ + id: "obs-1", + signal: "看重毛利率", + evidenceQuote: "毛利率是核心", + subjectHint: "毛利率", + confidence: 0.9, + referencedDocumentIds: ["doc-1"], + }) + }) +}) diff --git a/src/domains/memory/distill-workflow.ts b/src/domains/memory/distill-workflow.ts new file mode 100644 index 0000000..c491a71 --- /dev/null +++ b/src/domains/memory/distill-workflow.ts @@ -0,0 +1,231 @@ +import "server-only" + +import type { WorkflowContext } from "@upstash/workflow" + +import { + DISTILL_BATCH_MAX, + DISTILL_CONSUMED_RETENTION_MS, + DISTILL_DEDUP_CANDIDATES_PER_KIND, +} from "./distill-config" +import { distillMemoryPass } from "./distill-model" +import { + summarizePayloadForContext, + type DistillObservationInput, + type DistillPassKind, + type ExistingMemoryContextItem, + type MemoryOperations, +} from "./distill-types" +import { resolveMemoryOperations } from "./resolve-operations" +import { tokenizeMemoryText } from "./search-index" +import { memoryService } from "./service" +import type { FluidMemoryKind } from "./types" +import { isFluidMemoryKind } from "./types" +import type { FluidMemoryItem, FluidObservation } from "@/infrastructure/db/schema" +import { logger } from "@/lib/logger" + +export type MemoryDistillPayload = { + readonly workspaceId: string +} + +type MemoryDistillWorkflowContext = Pick< + WorkflowContext, + "run" +> + +const PASS_KINDS: readonly { + readonly pass: DistillPassKind + readonly kinds: readonly FluidMemoryKind[] +}[] = [ + { pass: "indicator", kinds: ["indicator_pref"] }, + { pass: "experience", kinds: ["stance", "decision_rule"] }, + { pass: "entity", kinds: ["entity_of_interest"] }, +] + +export function normalizeMemoryDistillPayload( + raw: unknown, +): MemoryDistillPayload | null { + if (!raw || typeof raw !== "object") return null + const workspaceId = getNonEmptyString( + (raw as Record).workspaceId, + ) + if (!workspaceId) return null + return { workspaceId } +} + +/** + * Periodic distill: pending observations → three typed passes → resolve → + * write fluid_memory_items and mark the batch consumed. Capture never writes + * the permanent layer; this job is the only writer. + */ +export async function runMemoryDistillWorkflow(input: { + readonly context: MemoryDistillWorkflowContext + readonly payload: MemoryDistillPayload +}): Promise { + const { context, payload } = input + + const batch = await context.run("select-batch", () => + memoryService.listPendingObservations( + payload.workspaceId, + DISTILL_BATCH_MAX, + ), + ) + if (batch.length === 0) { + logger.info("memory: distill skipped — no pending observations", { + workspaceId: payload.workspaceId, + }) + return + } + + const observationInputs = batch.map(toDistillObservationInput) + const referencedDocumentIds = unionDocumentIds(batch) + const queryTokens = tokenizeBatch(observationInputs) + + const candidatesByKind = await context.run("load-candidates", async () => { + const result: Partial> = {} + for (const kind of [ + "indicator_pref", + "stance", + "decision_rule", + "entity_of_interest", + ] as const) { + result[kind] = await memoryService.findDedupCandidates( + payload.workspaceId, + kind, + queryTokens, + DISTILL_DEDUP_CANDIDATES_PER_KIND, + ) + } + return result + }) + + const passOperations: MemoryOperations[] = [] + for (const { pass, kinds } of PASS_KINDS) { + const existingItems = kinds.flatMap((kind) => + (candidatesByKind[kind] ?? []).flatMap((item) => { + const mapped = toExistingMemoryContextItem(item) + return mapped ? [mapped] : [] + }), + ) + const operations = await context.run(`distill-${pass}`, () => + distillMemoryPass({ + pass, + workspaceId: payload.workspaceId, + observations: observationInputs, + existingItems, + referencedDocumentIds, + }), + ) + // Null = model failure for this pass only; other passes still apply. + if (operations) passOperations.push(operations) + } + + if (passOperations.length === 0) { + logger.warn( + "memory: distill aborted — all passes failed; batch left pending", + { + workspaceId: payload.workspaceId, + batchSize: batch.length, + }, + ) + return + } + + const existingItemRefs = Object.values(candidatesByKind) + .flat() + .map((item) => ({ + id: item.id, + kind: item.kind, + status: item.status, + payload: item.payload, + })) + + const resolved = passOperations.flatMap((operations) => + resolveMemoryOperations({ + operations, + existingItems: existingItemRefs, + referencedDocumentIds, + }), + ) + + const applied = await context.run("apply-and-consume", () => + memoryService.applyDistillBatch({ + workspaceId: payload.workspaceId, + sourceMessageId: null, + operations: resolved, + observationIds: batch.map((row) => row.id), + }), + ) + + const deleted = await context.run("retention", () => + memoryService.deleteExpiredConsumedObservations( + new Date(Date.now() - DISTILL_CONSUMED_RETENTION_MS), + ), + ) + + logger.info("memory: distill workflow finished", { + workspaceId: payload.workspaceId, + batchSize: batch.length, + resolvedCount: resolved.length, + diffCount: applied.diffs.length, + consumedCount: applied.consumedCount, + retentionDeleted: deleted, + }) +} + +export function toDistillObservationInput( + row: FluidObservation, +): DistillObservationInput { + const documentIds = Array.isArray(row.referencedDocumentIds) + ? row.referencedDocumentIds.filter( + (id): id is string => typeof id === "string" && id.length > 0, + ) + : [] + return { + id: row.id, + signal: row.signal, + evidenceQuote: row.evidenceQuote, + subjectHint: row.subjectHint, + confidence: row.confidence, + referencedDocumentIds: documentIds, + } +} + +function toExistingMemoryContextItem( + item: FluidMemoryItem, +): ExistingMemoryContextItem | null { + if (!isFluidMemoryKind(item.kind)) return null + return { + id: item.id, + kind: item.kind, + abstractL0: item.abstractL0, + payloadSummary: summarizePayloadForContext(item.kind, item.payload), + } +} + +function tokenizeBatch( + observations: readonly DistillObservationInput[], +): string[] { + const text = observations + .map((observation) => + [observation.signal, observation.subjectHint ?? "", observation.evidenceQuote] + .filter((part) => part.length > 0) + .join(" "), + ) + .join(" ") + return tokenizeMemoryText(text).map((entry) => entry.token) +} + +function unionDocumentIds(rows: readonly FluidObservation[]): string[] { + const ids = new Set() + for (const row of rows) { + if (!Array.isArray(row.referencedDocumentIds)) continue + for (const id of row.referencedDocumentIds) { + if (typeof id === "string" && id.length > 0) ids.add(id) + } + } + return [...ids] +} + +function getNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null +} diff --git a/src/domains/memory/extract-workflow.test.ts b/src/domains/memory/extract-workflow.test.ts new file mode 100644 index 0000000..41205a5 --- /dev/null +++ b/src/domains/memory/extract-workflow.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest" + +import { normalizeMemoryExtractPayload } from "./extract-workflow" + +describe("normalizeMemoryExtractPayload", () => { + it("accepts a complete payload", () => { + expect( + normalizeMemoryExtractPayload({ + workspaceId: "ws-1", + threadId: "th-1", + userMessageId: "u-1", + assistantMessageId: "a-1", + }), + ).toEqual({ + workspaceId: "ws-1", + threadId: "th-1", + userMessageId: "u-1", + assistantMessageId: "a-1", + }) + }) + + it("rejects missing or blank fields", () => { + expect(normalizeMemoryExtractPayload(null)).toBeNull() + expect( + normalizeMemoryExtractPayload({ + workspaceId: "ws-1", + threadId: "th-1", + userMessageId: "u-1", + }), + ).toBeNull() + expect( + normalizeMemoryExtractPayload({ + workspaceId: " ", + threadId: "th-1", + userMessageId: "u-1", + assistantMessageId: "a-1", + }), + ).toBeNull() + }) +}) diff --git a/src/domains/memory/extract-workflow.ts b/src/domains/memory/extract-workflow.ts index aa6b5ee..ec0a538 100644 --- a/src/domains/memory/extract-workflow.ts +++ b/src/domains/memory/extract-workflow.ts @@ -2,15 +2,9 @@ import "server-only" import type { WorkflowContext } from "@upstash/workflow" -import { extractMemoryOperations } from "./extraction-model" -import { - summarizePayloadForContext, - type ExistingMemoryContextItem, -} from "./prompts" -import { resolveMemoryOperations } from "./resolve-operations" -import { tokenizeMemoryText } from "./search-index" +import { triggerMemoryDistill } from "./distill-trigger" +import { captureObservations } from "./extraction-model" import { memoryService } from "./service" -import { fluidMemoryKinds, isFluidMemoryKind } from "./types" import { chatThreadService } from "@/domains/chat/thread-service" import { logger } from "@/lib/logger" @@ -26,15 +20,6 @@ type MemoryExtractWorkflowContext = Pick< "run" > -/** Prompt-context item that also carries status/payload for resolution. */ -type MemoryWorkflowItem = ExistingMemoryContextItem & { - readonly status: string - readonly payload: unknown -} - -/** Per-kind cap on lexical neighbors fed into the merge-decision prompt. */ -const DEDUP_CANDIDATES_PER_KIND = 8 - export function normalizeMemoryExtractPayload( raw: unknown, ): MemoryExtractPayload | null { @@ -50,6 +35,11 @@ export function normalizeMemoryExtractPayload( return { workspaceId, threadId, userMessageId, assistantMessageId } } +/** + * Per-turn coarse capture: load the turn → LLM observations → append-only + * insert into fluid_observations. Never writes fluid_memory_items (distill + * owns that). After persist, maybe-trigger distill when pending is high enough. + */ export async function runMemoryExtractWorkflow(input: { readonly context: MemoryExtractWorkflowContext readonly payload: MemoryExtractPayload @@ -77,73 +67,42 @@ export async function runMemoryExtractWorkflow(input: { } }) if (!turn) { - logger.warn("memory: extract skipped — turn messages not found", { + logger.warn("memory: capture skipped — turn messages not found", { workspaceId: payload.workspaceId, threadId: payload.threadId, }) return } - const existingItems = await context.run("retrieve-candidates", async () => { - const queryTokens = tokenizeMemoryText(turn.userText).map( - (entry) => entry.token, - ) - if (queryTokens.length === 0) return [] - - const byId = new Map() - for (const kind of fluidMemoryKinds) { - const items = await memoryService.findDedupCandidates( - payload.workspaceId, - kind, - queryTokens, - DEDUP_CANDIDATES_PER_KIND, - ) - for (const item of items) { - if (!isFluidMemoryKind(item.kind) || byId.has(item.id)) continue - byId.set(item.id, { - id: item.id, - kind: item.kind, - status: item.status, - payload: item.payload, - abstractL0: item.abstractL0, - payloadSummary: summarizePayloadForContext(item.kind, item.payload), - }) - } - } - return [...byId.values()] - }) - - const operations = await context.run("extract-operations", () => - extractMemoryOperations({ + const observations = await context.run("capture", () => + captureObservations({ workspaceId: payload.workspaceId, userText: turn.userText, assistantText: turn.assistantText, referencedDocumentIds: turn.referencedDocumentIds, - existingItems, }), ) - if (!operations) return + if (!observations) return - const applied = await context.run("apply-operations", async () => { - const resolved = resolveMemoryOperations({ - operations, - existingItems, + const inserted = await context.run("persist-observations", async () => { + if (observations.length === 0) return [] + return memoryService.insertObservations({ + workspaceId: payload.workspaceId, + sourceMessageId: payload.assistantMessageId, referencedDocumentIds: turn.referencedDocumentIds, + observations, }) - if (resolved.length === 0) return null - return memoryService.applyOperations( - payload.workspaceId, - payload.assistantMessageId, - resolved, - ) }) - logger.info("memory: extract workflow finished", { + await context.run("maybe-trigger-distill", async () => { + await triggerMemoryDistill({ workspaceId: payload.workspaceId }) + }) + + logger.info("memory: capture workflow finished", { workspaceId: payload.workspaceId, threadId: payload.threadId, assistantMessageId: payload.assistantMessageId, - candidateCount: existingItems.length, - appliedOperations: applied?.map((operation) => operation.op) ?? [], + observationCount: inserted.length, }) } diff --git a/src/domains/memory/extraction-model.ts b/src/domains/memory/extraction-model.ts index fa49011..e013e36 100644 --- a/src/domains/memory/extraction-model.ts +++ b/src/domains/memory/extraction-model.ts @@ -3,11 +3,11 @@ import "server-only" import { generateObject } from "ai" import { - buildMemoryExtractionPrompt, - memoryOperationsSchema, - type ExistingMemoryContextItem, - type MemoryOperations, -} from "./prompts" + captureOutputSchema, + type CaptureOutput, + type CapturedObservation, +} from "./observation-types" +import { buildCapturePrompt } from "./prompts" import { CHAT_MODEL } from "@/lib/ai" import { summarizeUnknownError } from "@/lib/format-log-value" import { logger } from "@/lib/logger" @@ -15,35 +15,34 @@ import { logger } from "@/lib/logger" const MEMORY_EXTRACTION_MODEL = process.env.MEMORY_EXTRACTION_MODEL ?? CHAT_MODEL /** - * One structured-output call: turn + existing active memories in, typed - * operations out. Best-effort by design — this runs as a background job, - * so a model failure skips the turn (logged) instead of degrading through - * fallbacks; the insight typically resurfaces in a later turn. + * One structured-output call: conversation turn in, raw observations out. + * Best-effort — this runs as a background job, so a model failure skips the + * turn (logged) instead of degrading through fallbacks; the clue typically + * resurfaces in a later turn. */ -export async function extractMemoryOperations(input: { +export async function captureObservations(input: { readonly workspaceId: string readonly userText: string readonly assistantText: string readonly referencedDocumentIds: readonly string[] - readonly existingItems: readonly ExistingMemoryContextItem[] -}): Promise { +}): Promise { try { const response = await generateObject({ model: MEMORY_EXTRACTION_MODEL, - schema: memoryOperationsSchema, + schema: captureOutputSchema, messages: [ { role: "user", - content: buildMemoryExtractionPrompt(input), + content: buildCapturePrompt(input), }, ], }) - return response.object + const output: CaptureOutput = response.object + return output.observations } catch (error) { - logger.warn("memory: extraction model call failed; skipping turn", { + logger.warn("memory: capture model call failed; skipping turn", { workspaceId: input.workspaceId, model: MEMORY_EXTRACTION_MODEL, - existingItemCount: input.existingItems.length, error: summarizeUnknownError(error), }) return null diff --git a/src/domains/memory/observation-types.test.ts b/src/domains/memory/observation-types.test.ts new file mode 100644 index 0000000..0a99eb6 --- /dev/null +++ b/src/domains/memory/observation-types.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest" + +import { + captureOutputSchema, + capturedObservationSchema, +} from "./observation-types" + +describe("capturedObservationSchema", () => { + it("accepts a full observation", () => { + const parsed = capturedObservationSchema.parse({ + signal: "看重毛利率", + evidenceQuote: "毛利率是核心", + subjectHint: "毛利率", + confidence: 0.9, + }) + expect(parsed.subjectHint).toBe("毛利率") + expect(parsed).not.toHaveProperty("kindHint") + }) + + it("coerces null subjectHint to undefined (single preprocess)", () => { + const parsed = capturedObservationSchema.parse({ + signal: "长期持有", + evidenceQuote: "我做长期投资", + subjectHint: null, + confidence: 1, + }) + expect(parsed.subjectHint).toBeUndefined() + }) + + it("strips unknown kindHint if the model still emits it", () => { + const parsed = capturedObservationSchema.parse({ + signal: "看重毛利率", + evidenceQuote: "毛利率是核心", + kindHint: "indicator", + confidence: 0.9, + }) + expect(parsed).not.toHaveProperty("kindHint") + }) + + it("rejects empty signal or evidenceQuote", () => { + expect(() => + capturedObservationSchema.parse({ + signal: "", + evidenceQuote: "x", + confidence: 0.5, + }), + ).toThrow() + expect(() => + capturedObservationSchema.parse({ + signal: "x", + evidenceQuote: "", + confidence: 0.5, + }), + ).toThrow() + }) +}) + +describe("captureOutputSchema", () => { + it("defaults missing observations to empty array", () => { + expect(captureOutputSchema.parse({})).toEqual({ observations: [] }) + }) + + it("parses a batch of observations", () => { + const parsed = captureOutputSchema.parse({ + observations: [ + { + signal: "跟踪英伟达", + evidenceQuote: "英伟达一直在跟踪", + subjectHint: "英伟达", + confidence: 0.8, + }, + ], + }) + expect(parsed.observations).toHaveLength(1) + expect(parsed.observations[0]?.subjectHint).toBe("英伟达") + expect(parsed.observations[0]).not.toHaveProperty("kindHint") + }) +}) diff --git a/src/domains/memory/observation-types.ts b/src/domains/memory/observation-types.ts new file mode 100644 index 0000000..63f7598 --- /dev/null +++ b/src/domains/memory/observation-types.ts @@ -0,0 +1,50 @@ +import { z } from "zod" + +/** + * Coarse-capture contract for the raw observation layer. + * + * Capture records durable USER clues (what the user cares about) only. + * It does not classify into final memory kinds, does not dedup, and never + * writes `fluid_memory_items`. `subjectHint` is an optional topic anchor + * for later clustering — distill owns authoritative typing and merge. + */ + +/** Single concentrated null→undefined coerce for optional capture fields. */ +function nullToUndefined(value: unknown): unknown { + return value === null ? undefined : value +} + +export const capturedObservationSchema = z.object({ + signal: z + .string() + .min(1) + .describe( + "One durable clue about what the USER cares about, in the user's language.", + ), + evidenceQuote: z + .string() + .min(1) + .describe("Short verbatim snippet from the USER turn that supports signal."), + subjectHint: z.preprocess( + nullToUndefined, + z + .string() + .min(1) + .optional() + .describe( + "Optional short topic anchor (metric name, company, topic). Prefer omit when none.", + ), + ), + confidence: z + .number() + .min(0) + .max(1) + .describe("How explicitly the user stated this (1 = explicit)."), +}) + +export const captureOutputSchema = z.object({ + observations: z.array(capturedObservationSchema).default([]), +}) + +export type CapturedObservation = z.infer +export type CaptureOutput = z.infer diff --git a/src/domains/memory/prompts.test.ts b/src/domains/memory/prompts.test.ts index 2ae15ff..2f7c83f 100644 --- a/src/domains/memory/prompts.test.ts +++ b/src/domains/memory/prompts.test.ts @@ -1,20 +1,38 @@ import { describe, expect, it } from "vitest" -import { buildMemoryExtractionPrompt } from "./prompts" +import { buildCapturePrompt } from "./prompts" -describe("buildMemoryExtractionPrompt", () => { - const prompt = buildMemoryExtractionPrompt({ +describe("buildCapturePrompt", () => { + const prompt = buildCapturePrompt({ userText: "毛利率是核心。", assistantText: "明白。", referencedDocumentIds: ["doc-1"], - existingItems: [], }) - it("keeps main instructions domain-agnostic", () => { + it("keeps main instructions domain-agnostic and capture-only", () => { const main = prompt.slice(0, prompt.indexOf("## Illustrative examples")) expect(main).not.toMatch(/gross margin|市盈率|\bPE\b|Fed|美联储|NVIDIA|英伟达/i) + expect(main).toContain("RAW OBSERVATIONS") + expect(main).toContain("Do NOT classify observations into those kinds") + expect(main).toContain("Do NOT emit any") + expect(main).toContain("kind / type / category field") + expect(main).toContain("Do NOT invent") + expect(main).toContain("create/merge/deprecate operations") + expect(main).not.toContain("kindHint") + expect(main).not.toContain("EXISTING MEMORIES") + expect(main).not.toContain("indicatorPrefs") + expect(main).not.toContain("decisionRules") expect(main).toContain("Write every free-text value") - expect(main).toMatch(/same language the\s+USER wrote/) + expect(main).toMatch(/same\s+language the USER wrote/) + }) + + it("output schema has no early kind classification field", () => { + const schema = prompt.slice(prompt.indexOf("## Output JSON schema")) + expect(schema).not.toContain("kindHint") + expect(schema).toContain("subjectHint") + expect(schema).toContain("signal") + expect(schema).toContain("evidenceQuote") + expect(schema).toContain("confidence") }) it("keeps illustrative examples in a separate section", () => { @@ -28,8 +46,9 @@ describe("buildMemoryExtractionPrompt", () => { expect(examples).toContain("do not force the conversation into this domain") }) - it("still injects turn context after the fixed blocks", () => { + it("injects turn context and referenced docs; no existing-memory block", () => { expect(prompt).toContain("[user]\n毛利率是核心。") expect(prompt).toContain("doc-1") + expect(prompt).not.toContain("## EXISTING MEMORIES") }) }) diff --git a/src/domains/memory/prompts.ts b/src/domains/memory/prompts.ts index e2fbeb0..2f12965 100644 --- a/src/domains/memory/prompts.ts +++ b/src/domains/memory/prompts.ts @@ -1,235 +1,80 @@ -import { z } from "zod" - -import { - decisionRulePayloadSchema, - entityOfInterestPayloadSchema, - indicatorPreferencePayloadSchema, - stancePayloadSchema, - type FluidMemoryKind, -} from "./types" - /** - * LLM contract for fluid-memory extraction. + * Coarse-capture prompt for the raw observation layer. * - * Design borrowed from OpenViking's session-commit extraction (schema-driven - * typed operations, prefetch-then-decide), reimplemented as a single - * structured-output call: the model sees the turn plus lexically retrieved - * dedup candidates and directly outputs per-kind operations - * (create / skip / merge / deprecate), mirroring OpenViking's generated - * operations model without its ReAct tool loop. + * Capture extracts durable USER points of concern only. It does not classify + * into final memory kinds, does not dedup against existing items, and does not + * emit create/merge/deprecate decisions — those belong to distill. */ -const decisionSchema = z.object({ - op: z.enum(["create", "skip", "merge", "deprecate"]), - targetItemId: z.preprocess( - (value) => (value === null ? undefined : value), - z - .string() - .optional() - .describe( - "Required for merge/deprecate: the id of the existing memory item this operation targets. Omit for create/skip.", - ), - ), - reason: z.preprocess( - (value) => (value === null ? undefined : value), - z - .string() - .optional() - .describe("Short justification, especially for skip/merge/deprecate."), - ), -}) - -const memorySidecarFields = { - abstractL0: z - .string() - .min(1) - .describe("One line, <= 30 words: the essence of this insight."), - overviewL1: z - .string() - .min(1) - .describe("2-3 sentences: what it means and when it applies."), - confidence: z - .number() - .min(0) - .max(1) - .describe("How explicitly the user stated this (1 = explicit)."), - decision: decisionSchema, -} - -const stanceEntrySchema = z.preprocess((value) => { - if (!value || typeof value !== "object" || Array.isArray(value)) return value - const record = value as Record - // Models sometimes emit "name" for a stance; the contract field is "statement". - if ( - (typeof record.statement !== "string" || record.statement.length === 0) && - typeof record.name === "string" && - record.name.length > 0 - ) { - const { name, ...rest } = record - return { ...rest, statement: name } - } - return value -}, stancePayloadSchema.extend(memorySidecarFields)) - -const entityEntrySchema = z.preprocess((value) => { - if (!value || typeof value !== "object" || Array.isArray(value)) return value - const record = value as Record - // Keep provenance reason required; if the model omitted it, fall back to L0. - if ( - (typeof record.reason !== "string" || record.reason.length === 0) && - typeof record.abstractL0 === "string" && - record.abstractL0.length > 0 - ) { - return { ...record, reason: record.abstractL0 } - } - return value -}, entityOfInterestPayloadSchema.extend(memorySidecarFields)) - -export const memoryOperationsSchema = z.object({ - indicatorPrefs: z - .array(indicatorPreferencePayloadSchema.extend(memorySidecarFields)) - .default([]), - stances: z.array(stanceEntrySchema).default([]), - decisionRules: z - .array(decisionRulePayloadSchema.extend(memorySidecarFields)) - .default([]), - entities: z.array(entityEntrySchema).default([]), -}) - -export type MemoryOperations = z.infer - -export type ExistingMemoryContextItem = { - readonly id: string - readonly kind: FluidMemoryKind - readonly abstractL0: string - readonly payloadSummary: string -} - /** Structural output shape only — no domain content. */ -const OUTPUT_SCHEMA_BLOCK = `{ - "indicatorPrefs": [{ - "name": "string", - "aliases": ["string"], - "definition": "string", - "polarity": "higher_better|lower_better|context", - "importance": "core|secondary", - "formulaHint": "string (optional — omit if none)", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }], - "stances": [{ - "statement": "string (the stance text; do not use a name field)", - "scope": "string", - "rationale": "string", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }], - "decisionRules": [{ - "when": "string", - "then": "string", - "priority": "high|medium|low", - "rationale": "string", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }], - "entities": [{ - "name": "string", - "ticker": "string optional", - "aliases": ["string"], - "knowhereDocumentIds": ["only ids listed under REFERENCED DOCUMENT IDS"], - "reason": "string", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } +const CAPTURE_OUTPUT_SCHEMA_BLOCK = `{ + "observations": [{ + "signal": "string — one durable clue about what the USER cares about", + "evidenceQuote": "string — short verbatim USER snippet supporting signal", + "subjectHint": "string optional — topic anchor (metric name / company / topic)", + "confidence": 0.0 }] }` /** - * Illustrative only — kept separate from the main instructions so the model - * does not treat these domain phrases as required vocabulary. - * Finance is the first vertical; add other industry blocks here later if needed. + * Illustrative only — kept separate so the model does not treat these domain + * phrases as required vocabulary. */ const ILLUSTRATIVE_EXAMPLES_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) -These show shape and judgement only. Extract whatever the user actually said; -do not force the conversation into this domain or these metric names. +These show shape and judgement only. Capture whatever the user actually said; +do not force the conversation into this domain. -- indicatorPref: user says a named metric they repeatedly use to judge quality - (shape: name + short definition + polarity + importance). Same idea applies - outside finance (any recurring evaluation metric). -- stance: user states a durable judgement frame that changes how evidence is - weighted (e.g. long-horizon vs short-horizon). -- decisionRule: user states a reusable when → then discipline over their metrics. -- entity: user says they actively track a named company/issuer and why. -- skip: a one-off factual question about a page/number in a document, small talk, - or an assistant suggestion the user did not endorse.` +- User names a recurring evaluation metric → one observation (signal + quote). +- User states a durable judgement frame (e.g. long-horizon) → one observation. +- User states a reusable when → then discipline → one observation. +- User says they actively track a named company → one observation. +- Skip: a one-off factual question about a page/number, small talk, or an + assistant suggestion the user did not endorse.` -/** Domain-agnostic extraction instructions. */ -const MAIN_INSTRUCTIONS_BLOCK = `You maintain a user's FLUID MEMORY: durable insights about how this user thinks, extracted from their conversation with an AI analyst. +const MAIN_INSTRUCTIONS_BLOCK = `You capture RAW OBSERVATIONS for a user's fluid memory pipeline. -Document facts live elsewhere (crystal memory). Never extract document facts, retrieved numbers, or page content as fluid memory. +These are cheap, high-recall clues about what the USER cares about. A later +distill step will decide final kinds (indicator / rule / stance / entity) and +merge them. Do NOT classify observations into those kinds. Do NOT emit any +kind / type / category field. Do NOT deduplicate. Do NOT invent +create/merge/deprecate operations. -## What to extract +Document facts live elsewhere (crystal memory). Never capture document facts, +retrieved numbers, or page content as observations. -Extract ONLY these four kinds, and ONLY when the turn gives real evidence from the USER: +## What to capture -- indicatorPrefs — recurring metrics or criteria the user uses to evaluate things. - Fields: name, aliases, definition, polarity (higher_better | lower_better | context), - importance (core | secondary), optional formulaHint. -- stances — durable positions that shape how the user weighs evidence. - Fields: statement (required; do not invent a "name" field), scope, rationale. -- decisionRules — reusable when → then disciplines the user stated or clearly endorsed. - Fields: when, then, priority (high | medium | low), rationale. -- entities — named subjects the user is actively tracking. - Fields: name, optional ticker, aliases, reason (required), knowhereDocumentIds - (only from REFERENCED DOCUMENT IDS below; never invent ids). +From the USER turn only, emit zero or more observations when there is real +evidence of a durable, reusable point of concern: + +- signal — one short clue in the user's language (what to remember later). +- evidenceQuote — a short verbatim snippet from the USER turn that supports it. +- subjectHint — optional short topic anchor (metric name, company, topic). Prefer omit when none. +- confidence — 1 only when the user stated it explicitly. ## Language - Keep this instruction set and enum/field names in English. -- Write every free-text value (name, definition, statement, when/then, reason, - abstractL0, overviewL1, aliases the user used, etc.) in the same language the - USER wrote in this turn. Do not translate the user's terms into English unless - the user themselves used English. +- Write every free-text value (signal, evidenceQuote, subjectHint) in the same + language the USER wrote in this turn. Do not translate the user's terms into + English unless the user themselves used English. -## Decision rules +## Judgement -- Extract only durable, reusable insights about the USER. -- Skip one-off questions, document facts, small talk, and assistant claims the user did not endorse. -- For every candidate, choose exactly one op against EXISTING MEMORIES: - - create — genuinely new - - skip — already covered, or too weak/ephemeral - - merge — same insight refined; emit the full merged fields and set targetItemId - - deprecate — user explicitly reversed a stored item; set targetItemId -- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. -- Prefer one record per insight. If a preference already encodes how a metric should be read, do not also invent a near-duplicate decisionRule unless the user stated an explicit when → then action. -- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when the user stated it explicitly. +- Capture only durable, reusable clues about what the USER cares about. +- Skip one-off questions, document facts, small talk, and assistant claims the + user did not endorse. +- Prefer atomic clues: one observation per distinct clue. Do not merge unrelated + ideas into one signal. - Omit optional fields instead of setting them to null. -- If nothing is worth remembering, return all four arrays empty.` +- If nothing is worth capturing, return {"observations": []}.` -export function buildMemoryExtractionPrompt(input: { +export function buildCapturePrompt(input: { readonly userText: string readonly assistantText: string readonly referencedDocumentIds: readonly string[] - readonly existingItems: readonly ExistingMemoryContextItem[] }): string { - const existingBlock = - input.existingItems.length === 0 - ? "(no existing memories yet)" - : input.existingItems - .map( - (item) => - `- [${item.kind}] id=${item.id} :: ${item.abstractL0} :: ${item.payloadSummary}`, - ) - .join("\n") - const documentsBlock = input.referencedDocumentIds.length === 0 ? "(no documents referenced in this turn)" @@ -241,11 +86,7 @@ ${ILLUSTRATIVE_EXAMPLES_BLOCK} ## Output JSON schema (follow exactly; do not invent fields) -${OUTPUT_SCHEMA_BLOCK} - -## EXISTING MEMORIES - -${existingBlock} +${CAPTURE_OUTPUT_SCHEMA_BLOCK} ## REFERENCED DOCUMENT IDS @@ -259,27 +100,3 @@ ${input.userText} [assistant] ${input.assistantText}` } - -export function summarizePayloadForContext( - kind: FluidMemoryKind, - payload: unknown, -): string { - if (!payload || typeof payload !== "object") return "" - const record = payload as Record - switch (kind) { - case "indicator_pref": - return [record.name, record.definition] - .filter((part) => typeof part === "string" && part.length > 0) - .join(" — ") - case "stance": - return typeof record.statement === "string" ? record.statement : "" - case "decision_rule": - return [record.when, record.then] - .filter((part) => typeof part === "string" && part.length > 0) - .join(" => ") - case "entity_of_interest": - return [record.name, record.ticker] - .filter((part) => typeof part === "string" && part.length > 0) - .join(" ") - } -} diff --git a/src/domains/memory/repository.ts b/src/domains/memory/repository.ts index f7f6b79..3d36662 100644 --- a/src/domains/memory/repository.ts +++ b/src/domains/memory/repository.ts @@ -1,8 +1,9 @@ import "server-only" -import { and, eq, inArray, sql } from "drizzle-orm" +import { and, asc, count, eq, inArray, lt, sql } from "drizzle-orm" import { Effect } from "effect" +import type { CapturedObservation } from "./observation-types" import { toDiffOperation, type ResolvedMemoryOperation } from "./resolve-operations" import { buildMemoryItemTokens } from "./search-index" import type { @@ -10,15 +11,31 @@ import type { FluidMemoryPayload, MemoryDiffOperation, } from "./types" -import { DbClient } from "@/infrastructure/db" +import { DbClient, type Db } from "@/infrastructure/db" import { fluidMemoryItems, fluidMemoryTokens, + fluidObservations, memoryDiffs, type FluidMemoryItem, + type FluidObservation, type NewFluidMemoryToken, } from "@/infrastructure/db/schema" +export type InsertObservationsInput = { + readonly workspaceId: string + readonly sourceMessageId: string | null + readonly referencedDocumentIds: readonly string[] + readonly observations: readonly CapturedObservation[] +} + +export type ApplyDistillBatchInput = { + readonly workspaceId: string + readonly sourceMessageId: string | null + readonly operations: readonly ResolvedMemoryOperation[] + readonly observationIds: readonly string[] +} + type MemoryRepository = { readonly findDedupCandidatesEffect: ( workspaceId: string, @@ -26,15 +43,35 @@ type MemoryRepository = { tokens: readonly string[], limit: number, ) => Effect.Effect - readonly applyOperationsEffect: ( + readonly insertObservationsEffect: ( + input: InsertObservationsInput, + ) => Effect.Effect + readonly countPendingObservationsEffect: ( workspaceId: string, - sourceMessageId: string | null, - operations: readonly ResolvedMemoryOperation[], - ) => Effect.Effect + ) => Effect.Effect + readonly listPendingObservationsEffect: ( + workspaceId: string, + limit: number, + ) => Effect.Effect + readonly applyDistillBatchEffect: ( + input: ApplyDistillBatchInput, + ) => Effect.Effect< + { + readonly diffs: readonly MemoryDiffOperation[] + readonly consumedCount: number + }, + never, + DbClient + > + readonly deleteExpiredConsumedObservationsEffect: ( + olderThan: Date, + ) => Effect.Effect } type RawRowsResult = readonly Row[] | { readonly rows: readonly Row[] } +type TxClient = Parameters[0]>[0] + /** * Retrieve the most lexically-similar active items of one kind, ranked by * idf-weighted token overlap computed entirely in SQL. Common tokens (high @@ -92,145 +129,272 @@ const findDedupCandidatesEffect: MemoryRepository["findDedupCandidatesEffect"] = }) }) -const applyOperationsEffect: MemoryRepository["applyOperationsEffect"] = ( - workspaceId, - sourceMessageId, - operations, +/** + * Append-only write of coarse-capture clues. Never touches fluid_memory_items. + * Empty input is a no-op (returns []). Status is always `pending`. + */ +const insertObservationsEffect: MemoryRepository["insertObservationsEffect"] = ( + input, +) => + Effect.gen(function* () { + const db = yield* DbClient + if (input.observations.length === 0) return [] + + const documentIds = [...input.referencedDocumentIds] + return yield* Effect.promise(() => + db + .insert(fluidObservations) + .values( + input.observations.map((observation) => ({ + workspaceId: input.workspaceId, + sourceMessageId: input.sourceMessageId, + signal: observation.signal, + evidenceQuote: observation.evidenceQuote, + subjectHint: observation.subjectHint ?? null, + referencedDocumentIds: documentIds, + confidence: observation.confidence, + status: "pending", + })), + ) + .returning(), + ) + }) + +const countPendingObservationsEffect: MemoryRepository["countPendingObservationsEffect"] = + (workspaceId) => + Effect.gen(function* () { + const db = yield* DbClient + const rows = yield* Effect.promise(() => + db + .select({ value: count() }) + .from(fluidObservations) + .where( + and( + eq(fluidObservations.workspaceId, workspaceId), + eq(fluidObservations.status, "pending"), + ), + ), + ) + return Number(rows[0]?.value ?? 0) + }) + +/** + * Oldest-first pending batch for distill. Concurrency across distill runs for + * the same workspace is primarily gated by trigger cooldown + bucketed + * workflowRunId; consume below is conditional on status still being pending. + */ +const listPendingObservationsEffect: MemoryRepository["listPendingObservationsEffect"] = + (workspaceId, limit) => + Effect.gen(function* () { + const db = yield* DbClient + if (limit <= 0) return [] + return yield* Effect.promise(() => + db + .select() + .from(fluidObservations) + .where( + and( + eq(fluidObservations.workspaceId, workspaceId), + eq(fluidObservations.status, "pending"), + ), + ) + .orderBy(asc(fluidObservations.createdAt)) + .limit(limit), + ) + }) + +const applyDistillBatchEffect: MemoryRepository["applyDistillBatchEffect"] = ( + input, ) => Effect.gen(function* () { const db = yield* DbClient return yield* Effect.promise(() => db.transaction(async (tx) => { - const diffOperations: MemoryDiffOperation[] = [] - - for (const operation of operations) { - switch (operation.op) { - case "create": { - const [inserted] = await tx - .insert(fluidMemoryItems) - .values({ - workspaceId, - kind: operation.kind, - payload: operation.payload, - abstractL0: operation.abstractL0, - overviewL1: operation.overviewL1, - sourceMessageId, - confidence: operation.confidence, - status: "active", - }) - .returning() - if (inserted?.id) { - const tokenRows = tokenRowsFor( - workspaceId, - inserted.id, - operation.kind, - operation.payload, - ) - if (tokenRows.length > 0) { - await tx.insert(fluidMemoryTokens).values(tokenRows) - } - } - diffOperations.push(toDiffOperation(operation, inserted?.id)) - break - } - case "merge": { - const [updated] = await tx - .update(fluidMemoryItems) - .set({ - payload: operation.payload, - abstractL0: operation.abstractL0, - overviewL1: operation.overviewL1, - confidence: operation.confidence, - sourceMessageId, - version: sql`${fluidMemoryItems.version} + 1`, - updatedAt: sql`now()`, - }) - .where( - and( - eq(fluidMemoryItems.id, operation.targetItemId), - eq(fluidMemoryItems.status, "active"), - ), - ) - .returning({ id: fluidMemoryItems.id }) - if (updated) { - await tx - .delete(fluidMemoryTokens) - .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) - const tokenRows = tokenRowsFor( - workspaceId, - operation.targetItemId, - operation.kind, - operation.payload, - ) - if (tokenRows.length > 0) { - await tx.insert(fluidMemoryTokens).values(tokenRows) - } - } - diffOperations.push( - updated - ? toDiffOperation(operation) - : { - op: "skip", - kind: operation.kind, - summary: operation.summary, - reason: "merge target no longer active", - }, - ) - break - } - case "deprecate": { - const [updated] = await tx - .update(fluidMemoryItems) - .set({ - status: "deprecated", - updatedAt: sql`now()`, - }) - .where( - and( - eq(fluidMemoryItems.id, operation.targetItemId), - eq(fluidMemoryItems.status, "active"), - ), - ) - .returning({ id: fluidMemoryItems.id }) - if (updated) { - await tx - .delete(fluidMemoryTokens) - .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) - } - diffOperations.push( - updated - ? toDiffOperation(operation) - : { - op: "skip", - kind: operation.kind, - summary: operation.summary, - reason: "deprecate target no longer active", - }, - ) - break - } - case "skip": - diffOperations.push(toDiffOperation(operation)) - break - } + const diffs = await writeOperations( + tx, + input.workspaceId, + input.sourceMessageId, + input.operations, + ) + let consumedCount = 0 + if (input.observationIds.length > 0) { + const consumed = await tx + .update(fluidObservations) + .set({ + status: "consumed", + consumedAt: sql`now()`, + }) + .where( + and( + eq(fluidObservations.workspaceId, input.workspaceId), + eq(fluidObservations.status, "pending"), + inArray(fluidObservations.id, [...input.observationIds]), + ), + ) + .returning({ id: fluidObservations.id }) + consumedCount = consumed.length } + return { diffs, consumedCount } + }), + ) + }) + +const deleteExpiredConsumedObservationsEffect: MemoryRepository["deleteExpiredConsumedObservationsEffect"] = + (olderThan) => + Effect.gen(function* () { + const db = yield* DbClient + const deleted = yield* Effect.promise(() => + db + .delete(fluidObservations) + .where( + and( + eq(fluidObservations.status, "consumed"), + lt(fluidObservations.createdAt, olderThan), + ), + ) + .returning({ id: fluidObservations.id }), + ) + return deleted.length + }) + +export const memoryRepository: MemoryRepository = { + findDedupCandidatesEffect, + insertObservationsEffect, + countPendingObservationsEffect, + listPendingObservationsEffect, + applyDistillBatchEffect, + deleteExpiredConsumedObservationsEffect, +} + +async function writeOperations( + tx: TxClient, + workspaceId: string, + sourceMessageId: string | null, + operations: readonly ResolvedMemoryOperation[], +): Promise { + const diffOperations: MemoryDiffOperation[] = [] - if (diffOperations.length > 0) { - await tx.insert(memoryDiffs).values({ + for (const operation of operations) { + switch (operation.op) { + case "create": { + const [inserted] = await tx + .insert(fluidMemoryItems) + .values({ workspaceId, + kind: operation.kind, + payload: operation.payload, + abstractL0: operation.abstractL0, + overviewL1: operation.overviewL1, sourceMessageId, - operations: [...diffOperations], + confidence: operation.confidence, + status: "active", }) + .returning() + if (inserted?.id) { + const tokenRows = tokenRowsFor( + workspaceId, + inserted.id, + operation.kind, + operation.payload, + ) + if (tokenRows.length > 0) { + await tx.insert(fluidMemoryTokens).values(tokenRows) + } + } + diffOperations.push(toDiffOperation(operation, inserted?.id)) + break + } + case "merge": { + const [updated] = await tx + .update(fluidMemoryItems) + .set({ + payload: operation.payload, + abstractL0: operation.abstractL0, + overviewL1: operation.overviewL1, + confidence: operation.confidence, + sourceMessageId, + version: sql`${fluidMemoryItems.version} + 1`, + updatedAt: sql`now()`, + }) + .where( + and( + eq(fluidMemoryItems.id, operation.targetItemId), + eq(fluidMemoryItems.status, "active"), + ), + ) + .returning({ id: fluidMemoryItems.id }) + if (updated) { + await tx + .delete(fluidMemoryTokens) + .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) + const tokenRows = tokenRowsFor( + workspaceId, + operation.targetItemId, + operation.kind, + operation.payload, + ) + if (tokenRows.length > 0) { + await tx.insert(fluidMemoryTokens).values(tokenRows) + } } + diffOperations.push( + updated + ? toDiffOperation(operation) + : { + op: "skip", + kind: operation.kind, + summary: operation.summary, + reason: "merge target no longer active", + }, + ) + break + } + case "deprecate": { + const [updated] = await tx + .update(fluidMemoryItems) + .set({ + status: "deprecated", + updatedAt: sql`now()`, + }) + .where( + and( + eq(fluidMemoryItems.id, operation.targetItemId), + eq(fluidMemoryItems.status, "active"), + ), + ) + .returning({ id: fluidMemoryItems.id }) + if (updated) { + await tx + .delete(fluidMemoryTokens) + .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) + } + diffOperations.push( + updated + ? toDiffOperation(operation) + : { + op: "skip", + kind: operation.kind, + summary: operation.summary, + reason: "deprecate target no longer active", + }, + ) + break + } + case "skip": + diffOperations.push(toDiffOperation(operation)) + break + } + } - return diffOperations - }), - ) - }) + if (diffOperations.length > 0) { + await tx.insert(memoryDiffs).values({ + workspaceId, + sourceMessageId, + operations: [...diffOperations], + }) + } -export const memoryRepository: MemoryRepository = { - findDedupCandidatesEffect, - applyOperationsEffect, + return diffOperations } function tokenRowsFor( diff --git a/src/domains/memory/resolve-operations.test.ts b/src/domains/memory/resolve-operations.test.ts index 9764ccf..403a79a 100644 --- a/src/domains/memory/resolve-operations.test.ts +++ b/src/domains/memory/resolve-operations.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" -import type { MemoryOperations } from "./prompts" +import type { MemoryOperations } from "./resolve-operations" import { resolveMemoryOperations, toDiffOperation, diff --git a/src/domains/memory/resolve-operations.ts b/src/domains/memory/resolve-operations.ts index ee1e6a4..833b955 100644 --- a/src/domains/memory/resolve-operations.ts +++ b/src/domains/memory/resolve-operations.ts @@ -1,5 +1,5 @@ +import type { MemoryOperations } from "./distill-types" import { buildMemoryItemTokens } from "./search-index" -import type { MemoryOperations } from "./prompts" import { parseFluidMemoryPayload, type FluidMemoryKind, @@ -7,6 +7,8 @@ import { type MemoryDiffOperation, } from "./types" +export type { MemoryOperations } from "./distill-types" + /** * Pure normalization from raw LLM operations to repository-ready * operations. The LLM output already passed zod validation; this layer @@ -14,12 +16,14 @@ import { * - merge/deprecate must target an existing active item of the same kind * (otherwise downgraded to skip — conservative, never fabricates) * - entity knowhereDocumentIds are intersected with the document ids - * actually referenced in the turn (the model cannot invent provenance) + * allowed for the batch (the model cannot invent provenance) * - create ignores any targetItemId the model may have emitted * - create/merge payloads must yield at least one lexical token, otherwise * the item could never be retrieved for later dedup * - merge unions aliases (and entity document ids) with the target so * prior search terms / provenance are not wiped by a partial rewrite + * + * Used by distill (not by per-turn capture). Capture only writes observations. */ export type ResolvedMemoryOperation = @@ -65,17 +69,6 @@ export type ExistingMemoryItemRef = { readonly payload?: unknown } -type CandidateEntry = { - readonly abstractL0: string - readonly overviewL1: string - readonly confidence: number - readonly decision: { - readonly op: "create" | "skip" | "merge" | "deprecate" - readonly targetItemId?: string - readonly reason?: string - } -} - const kindToArrayKey = { indicator_pref: "indicatorPrefs", stance: "stances", @@ -98,8 +91,7 @@ export function resolveMemoryOperations(input: { const resolved: ResolvedMemoryOperation[] = [] for (const kind of Object.keys(kindToArrayKey) as FluidMemoryKind[]) { - const entries = input.operations[kindToArrayKey[kind]] as readonly (CandidateEntry & - Record)[] + const entries = input.operations[kindToArrayKey[kind]] for (const entry of entries) { const summary = entry.abstractL0 @@ -132,7 +124,11 @@ export function resolveMemoryOperations(input: { }) continue } - const mergePayload = toPayload(kind, entry, allowedDocumentIds) + const mergePayload = toPayload( + kind, + entry as Record, + allowedDocumentIds, + ) if (!mergePayload) { resolved.push({ op: "skip", @@ -170,7 +166,11 @@ export function resolveMemoryOperations(input: { continue } - const createPayload = toPayload(kind, entry, allowedDocumentIds) + const createPayload = toPayload( + kind, + entry as Record, + allowedDocumentIds, + ) if (!createPayload) { resolved.push({ op: "skip", @@ -241,7 +241,7 @@ function isIndexable(kind: FluidMemoryKind, payload: FluidMemoryPayload): boolea } /** - * Merge replaces the stored payload, but the model only sees this turn. + * Merge replaces the stored payload, but the model only sees this batch. * Union aliases (and entity document ids) with the target so earlier search * terms / provenance survive a partial rewrite. */ diff --git a/src/domains/memory/service.ts b/src/domains/memory/service.ts index da772f6..41b8b5a 100644 --- a/src/domains/memory/service.ts +++ b/src/domains/memory/service.ts @@ -1,10 +1,16 @@ import "server-only" import { databaseRuntime } from "@/domains/workspace/database-runtime" -import { memoryRepository } from "./repository" -import type { ResolvedMemoryOperation } from "./resolve-operations" +import { + memoryRepository, + type ApplyDistillBatchInput, + type InsertObservationsInput, +} from "./repository" import type { FluidMemoryKind, MemoryDiffOperation } from "./types" -import type { FluidMemoryItem } from "@/infrastructure/db/schema" +import type { + FluidMemoryItem, + FluidObservation, +} from "@/infrastructure/db/schema" type MemoryService = { readonly findDedupCandidates: ( @@ -13,11 +19,21 @@ type MemoryService = { tokens: readonly string[], limit: number, ) => Promise - readonly applyOperations: ( + readonly insertObservations: ( + input: InsertObservationsInput, + ) => Promise + readonly countPendingObservations: (workspaceId: string) => Promise + readonly listPendingObservations: ( workspaceId: string, - sourceMessageId: string | null, - operations: readonly ResolvedMemoryOperation[], - ) => Promise + limit: number, + ) => Promise + readonly applyDistillBatch: (input: ApplyDistillBatchInput) => Promise<{ + readonly diffs: readonly MemoryDiffOperation[] + readonly consumedCount: number + }> + readonly deleteExpiredConsumedObservations: ( + olderThan: Date, + ) => Promise } const findDedupCandidates: MemoryService["findDedupCandidates"] = ( @@ -35,20 +51,38 @@ const findDedupCandidates: MemoryService["findDedupCandidates"] = ( ), ) -const applyOperations: MemoryService["applyOperations"] = ( +const insertObservations: MemoryService["insertObservations"] = (input) => + databaseRuntime.runPromise(memoryRepository.insertObservationsEffect(input)) + +const countPendingObservations: MemoryService["countPendingObservations"] = ( workspaceId, - sourceMessageId, - operations, ) => databaseRuntime.runPromise( - memoryRepository.applyOperationsEffect( - workspaceId, - sourceMessageId, - operations, - ), + memoryRepository.countPendingObservationsEffect(workspaceId), + ) + +const listPendingObservations: MemoryService["listPendingObservations"] = ( + workspaceId, + limit, +) => + databaseRuntime.runPromise( + memoryRepository.listPendingObservationsEffect(workspaceId, limit), ) +const applyDistillBatch: MemoryService["applyDistillBatch"] = (input) => + databaseRuntime.runPromise(memoryRepository.applyDistillBatchEffect(input)) + +const deleteExpiredConsumedObservations: MemoryService["deleteExpiredConsumedObservations"] = + (olderThan) => + databaseRuntime.runPromise( + memoryRepository.deleteExpiredConsumedObservationsEffect(olderThan), + ) + export const memoryService: MemoryService = { findDedupCandidates, - applyOperations, + insertObservations, + countPendingObservations, + listPendingObservations, + applyDistillBatch, + deleteExpiredConsumedObservations, } diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index a3c9805..cb767a0 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -476,3 +476,53 @@ export const memoryDiffs = pgTable( export type MemoryDiff = typeof memoryDiffs.$inferSelect; export type NewMemoryDiff = typeof memoryDiffs.$inferInsert; + +/** + * Append-only raw observation layer for fluid memory (L2 evidence). + * + * Each chat turn may write zero or more pending rows here via cheap capture. + * A later distill job consumes a batch, upserts typed items into + * `fluid_memory_items`, and marks these rows `consumed`. Capture never writes + * the distilled layer; distill is the only writer of permanent memory. + * + * Capture stores points of concern only — no early kind classification. + * `subject_hint` is an optional topic anchor for later clustering; distill + * owns the final kind and merge decision. + */ +export const fluidObservations = pgTable( + "fluid_observations", + { + id: uuid("id").primaryKey().defaultRandom(), + workspaceId: uuid("workspace_id") + .notNull() + .references(() => workspaces.id, { onDelete: "cascade" }), + sourceMessageId: uuid("source_message_id").references( + () => chatMessages.id, + { onDelete: "set null" }, + ), + signal: text("signal").notNull(), + evidenceQuote: text("evidence_quote").notNull(), + subjectHint: text("subject_hint"), + referencedDocumentIds: jsonb("referenced_document_ids") + .$type() + .notNull() + .default(sql`'[]'::jsonb`), + confidence: doublePrecision("confidence").notNull(), + status: text("status").notNull().default("pending"), + createdAt: timestamp("created_at", { withTimezone: true }) + .notNull() + .defaultNow(), + consumedAt: timestamp("consumed_at", { withTimezone: true }), + }, + (t) => [ + // Distill scan: pending rows for a workspace in capture order. + index("fluid_observations_workspace_status_created_idx").on( + t.workspaceId, + t.status, + t.createdAt, + ), + ], +); + +export type FluidObservation = typeof fluidObservations.$inferSelect; +export type NewFluidObservation = typeof fluidObservations.$inferInsert; From 823356c3cf842c73e247d31dd534b7990d5e980d Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Thu, 10 Sep 2026 10:49:03 +0800 Subject: [PATCH 2/4] feat(memory): enhance memory search functionality and integrate into agent harness - Added a new memory search tool to retrieve insights from fluid memory, allowing for more context-aware responses. - Updated the agent harness to utilize memory tools, ensuring that memory searches are prioritized before document retrieval. - Introduced memory citations in output manifests to track references from memory searches. - Enhanced tests to validate the new memory search behavior and its integration with existing functionalities. This update improves the agent's ability to leverage past interactions for more relevant and informed responses. --- drizzle/0016_wealthy_matthew_murdock.sql | 13 + drizzle/meta/0016_snapshot.json | 1550 +++++++++++++++++ drizzle/meta/_journal.json | 7 + scripts/guanxin-case/constants.ts | 8 + scripts/guanxin-case/ensure-workspace.mts | 21 + scripts/guanxin-case/sample-pilot.py | 91 + src/agent-harness/index.ts | 1 + src/agent-harness/memory-text.test.ts | 38 + src/agent-harness/memory-text.ts | 93 + src/agent-harness/runtime.test.ts | 190 +- src/agent-harness/runtime.ts | 179 +- src/agent-harness/types.ts | 38 + src/domains/chat/citations.test.ts | 15 + src/domains/chat/citations.ts | 1 + src/domains/chat/commit-turn.test.ts | 138 ++ src/domains/chat/commit-turn.ts | 131 ++ src/domains/chat/index.test.ts | 23 +- src/domains/chat/memory-tools.test.ts | 104 ++ src/domains/chat/memory-tools.ts | 66 + src/domains/chat/prompt.ts | 5 + src/domains/chat/route-answer.ts | 24 +- src/domains/chat/route-service.test.ts | 2 +- src/domains/chat/service.test.ts | 34 + src/domains/chat/service.ts | 2 +- src/domains/chat/types.ts | 1 + src/domains/memory/decay-candidates.test.ts | 69 + src/domains/memory/decay-candidates.ts | 64 + src/domains/memory/repository.ts | 84 +- src/domains/memory/resolve-operations.test.ts | 4 +- src/domains/memory/service.ts | 58 + src/domains/memory/types.ts | 11 + .../retrieval-activation/decay-score.test.ts | 98 ++ .../retrieval-activation/decay-score.ts | 66 + .../retrieval-activation/repository.test.ts | 138 ++ .../retrieval-activation/repository.ts | 119 ++ src/domains/retrieval-activation/service.ts | 45 + src/domains/retrieval-activation/types.ts | 17 + src/infrastructure/db/schema.ts | 60 +- 38 files changed, 3556 insertions(+), 52 deletions(-) create mode 100644 drizzle/0016_wealthy_matthew_murdock.sql create mode 100644 drizzle/meta/0016_snapshot.json create mode 100644 scripts/guanxin-case/constants.ts create mode 100644 scripts/guanxin-case/ensure-workspace.mts create mode 100644 scripts/guanxin-case/sample-pilot.py create mode 100644 src/agent-harness/memory-text.test.ts create mode 100644 src/agent-harness/memory-text.ts create mode 100644 src/domains/chat/commit-turn.test.ts create mode 100644 src/domains/chat/commit-turn.ts create mode 100644 src/domains/chat/memory-tools.test.ts create mode 100644 src/domains/chat/memory-tools.ts create mode 100644 src/domains/memory/decay-candidates.test.ts create mode 100644 src/domains/memory/decay-candidates.ts create mode 100644 src/domains/retrieval-activation/decay-score.test.ts create mode 100644 src/domains/retrieval-activation/decay-score.ts create mode 100644 src/domains/retrieval-activation/repository.test.ts create mode 100644 src/domains/retrieval-activation/repository.ts create mode 100644 src/domains/retrieval-activation/service.ts create mode 100644 src/domains/retrieval-activation/types.ts diff --git a/drizzle/0016_wealthy_matthew_murdock.sql b/drizzle/0016_wealthy_matthew_murdock.sql new file mode 100644 index 0000000..4d7a57e --- /dev/null +++ b/drizzle/0016_wealthy_matthew_murdock.sql @@ -0,0 +1,13 @@ +CREATE TABLE "retrieval_activations" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "unit_type" text NOT NULL, + "unit_ref" text NOT NULL, + "activation_count" integer DEFAULT 0 NOT NULL, + "last_activated_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "fluid_memory_items" ADD COLUMN "deactivation_reason" text;--> statement-breakpoint +ALTER TABLE "retrieval_activations" ADD CONSTRAINT "retrieval_activations_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "retrieval_activations_unit_idx" ON "retrieval_activations" USING btree ("workspace_id","unit_type","unit_ref"); \ No newline at end of file diff --git a/drizzle/meta/0016_snapshot.json b/drizzle/meta/0016_snapshot.json new file mode 100644 index 0000000..77ef531 --- /dev/null +++ b/drizzle/meta/0016_snapshot.json @@ -0,0 +1,1550 @@ +{ + "id": "55d73dc7-9826-4cb5-9bb6-3506b8fa338a", + "prevId": "be8deb1f-3ec8-4304-85d0-41403629970c", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_memory_items": { + "name": "fluid_memory_items", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "abstract_l0": { + "name": "abstract_l0", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "overview_l1": { + "name": "overview_l1", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deactivation_reason": { + "name": "deactivation_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "fluid_memory_items_workspace_status_idx": { + "name": "fluid_memory_items_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fluid_memory_items_workspace_kind_idx": { + "name": "fluid_memory_items_workspace_kind_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_memory_items_workspace_id_workspaces_id_fk": { + "name": "fluid_memory_items_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_memory_items_source_message_id_chat_messages_id_fk": { + "name": "fluid_memory_items_source_message_id_chat_messages_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_memory_tokens": { + "name": "fluid_memory_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "item_id": { + "name": "item_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "frequency": { + "name": "frequency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + } + }, + "indexes": { + "fluid_memory_tokens_lookup_idx": { + "name": "fluid_memory_tokens_lookup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fluid_memory_tokens_item_idx": { + "name": "fluid_memory_tokens_item_idx", + "columns": [ + { + "expression": "item_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_memory_tokens_workspace_id_workspaces_id_fk": { + "name": "fluid_memory_tokens_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_memory_tokens", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_memory_tokens_item_id_fluid_memory_items_id_fk": { + "name": "fluid_memory_tokens_item_id_fluid_memory_items_id_fk", + "tableFrom": "fluid_memory_tokens", + "tableTo": "fluid_memory_items", + "columnsFrom": [ + "item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_observations": { + "name": "fluid_observations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "signal": { + "name": "signal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "evidence_quote": { + "name": "evidence_quote", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "subject_hint": { + "name": "subject_hint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "referenced_document_ids": { + "name": "referenced_document_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "confidence": { + "name": "confidence", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "fluid_observations_workspace_status_created_idx": { + "name": "fluid_observations_workspace_status_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_observations_workspace_id_workspaces_id_fk": { + "name": "fluid_observations_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_observations", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_observations_source_message_id_chat_messages_id_fk": { + "name": "fluid_observations_source_message_id_chat_messages_id_fk", + "tableFrom": "fluid_observations", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_diffs": { + "name": "memory_diffs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memory_diffs_workspace_created_idx": { + "name": "memory_diffs_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_diffs_workspace_id_workspaces_id_fk": { + "name": "memory_diffs_workspace_id_workspaces_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memory_diffs_source_message_id_chat_messages_id_fk": { + "name": "memory_diffs_source_message_id_chat_messages_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parsed_document_sync_leases": { + "name": "parsed_document_sync_leases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_token": { + "name": "lease_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_reason": { + "name": "release_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parsed_document_sync_leases_token_idx": { + "name": "parsed_document_sync_leases_token_idx", + "columns": [ + { + "expression": "lease_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_active_idx": { + "name": "parsed_document_sync_leases_active_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_workspace_active_idx": { + "name": "parsed_document_sync_leases_workspace_active_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_document_active_idx": { + "name": "parsed_document_sync_leases_document_active_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parsed_document_sync_leases_workspace_id_workspaces_id_fk": { + "name": "parsed_document_sync_leases_workspace_id_workspaces_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parsed_document_sync_leases_source_id_sources_id_fk": { + "name": "parsed_document_sync_leases_source_id_sources_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.retrieval_activations": { + "name": "retrieval_activations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "unit_type": { + "name": "unit_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "unit_ref": { + "name": "unit_ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activation_count": { + "name": "activation_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_activated_at": { + "name": "last_activated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "retrieval_activations_unit_idx": { + "name": "retrieval_activations_unit_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "unit_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "unit_ref", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "retrieval_activations_workspace_id_workspaces_id_fk": { + "name": "retrieval_activations_workspace_id_workspaces_id_fk", + "tableFrom": "retrieval_activations", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 32ea955..a8f7671 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -113,6 +113,13 @@ "when": 1788418416499, "tag": "0015_bumpy_vulcan", "breakpoints": true + }, + { + "idx": 16, + "version": "7", + "when": 1788952172906, + "tag": "0016_wealthy_matthew_murdock", + "breakpoints": true } ] } \ No newline at end of file diff --git a/scripts/guanxin-case/constants.ts b/scripts/guanxin-case/constants.ts new file mode 100644 index 0000000..d4f6099 --- /dev/null +++ b/scripts/guanxin-case/constants.ts @@ -0,0 +1,8 @@ +/** + * Dedicated Notebook workspace for the 观心 cardiovascular case. + * workspaces 表没有 title 列，用稳定 userId 标识。 + */ +export const GUANXIN_CASE_USER_ID = "case:guanxin-cardiovascular" + +/** 方案约定的 pilot 规模（20–30）按 sheet×难度 6 层均分。 */ +export const PILOT_CASES_PER_STRATUM = 4 diff --git a/scripts/guanxin-case/ensure-workspace.mts b/scripts/guanxin-case/ensure-workspace.mts new file mode 100644 index 0000000..963bca6 --- /dev/null +++ b/scripts/guanxin-case/ensure-workspace.mts @@ -0,0 +1,21 @@ +/** + * Ensure the dedicated 观心 case workspace exists in the Notebook database. + * Requires DATABASE_URL (Notebook Neon/Postgres), not the Knowhere eval DSN. + * + * DATABASE_URL=... node --experimental-strip-types scripts/guanxin-case/ensure-workspace.mts + */ +import { workspaceService } from "../../src/domains/workspace/service.ts" +import { GUANXIN_CASE_USER_ID } from "./constants.ts" + +const workspace = await workspaceService.ensureWorkspace(GUANXIN_CASE_USER_ID) +console.log( + JSON.stringify( + { + userId: workspace.userId, + workspaceId: workspace.id, + namespace: workspace.namespace, + }, + null, + 2, + ), +) diff --git a/scripts/guanxin-case/sample-pilot.py b/scripts/guanxin-case/sample-pilot.py new file mode 100644 index 0000000..a743d4d --- /dev/null +++ b/scripts/guanxin-case/sample-pilot.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Stratified pilot sample from 观心 knowhere自测集.xlsx. + +Takes the first PILOT_CASES_PER_STRATUM rows (by seq_id) from each +(sheet, 难度) bucket. Does not call Knowhere or write the case workspace. +""" +from __future__ import annotations + +import json +import sys +from collections import defaultdict +from pathlib import Path + +import openpyxl + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_XLSX = Path( + "/Users/wuchengke/Desktop/skills-coding/观心2.0-RAG-v1.1-demo/knowhere自测集.xlsx" +) +DEFAULT_OUT = ROOT / ".tmp" / "guanxin-pilot-cases.json" +PILOT_CASES_PER_STRATUM = 4 + + +def load_rows(xlsx: Path) -> list[dict]: + wb = openpyxl.load_workbook(xlsx, read_only=True, data_only=True) + rows: list[dict] = [] + try: + for sheet_name in wb.sheetnames: + sheet_rows = list(wb[sheet_name].iter_rows(values_only=True)) + if not sheet_rows: + continue + header = [ + str(h).strip() if h is not None else f"col{i}" + for i, h in enumerate(sheet_rows[0]) + ] + for raw in sheet_rows[1:]: + if not raw or raw[0] in (None, ""): + continue + item = { + header[i]: (raw[i] if i < len(raw) else None) + for i in range(len(header)) + } + query = str(item.get("具体query") or "").strip() + if not query: + continue + rows.append( + { + "sheet": sheet_name, + "seq_id": str(item.get("seq_id") or ""), + "query": query, + "disease": str(item.get("具体疾病名称") or "").strip(), + "scene": str(item.get("应用场景-考察能力") or "").strip(), + "difficulty": str(item.get("难度") or "").strip(), + "input_type": str(item.get("输入类型") or "").strip(), + } + ) + finally: + wb.close() + return rows + + +def sample(rows: list[dict]) -> list[dict]: + buckets: dict[tuple[str, str], list[dict]] = defaultdict(list) + for row in rows: + buckets[(row["sheet"], row["difficulty"] or "?")].append(row) + picked: list[dict] = [] + for key in sorted(buckets): + group = sorted(buckets[key], key=lambda row: row["seq_id"]) + picked.extend(group[:PILOT_CASES_PER_STRATUM]) + return sorted(picked, key=lambda row: (row["sheet"], row["seq_id"])) + + +def main() -> None: + xlsx = Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_XLSX + out = Path(sys.argv[2]) if len(sys.argv) > 2 else DEFAULT_OUT + rows = load_rows(xlsx) + picked = sample(rows) + out.parent.mkdir(parents=True, exist_ok=True) + payload = { + "xlsx": str(xlsx), + "per_stratum": PILOT_CASES_PER_STRATUM, + "source_count": len(rows), + "pilot_count": len(picked), + "cases": picked, + } + out.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(f"wrote {len(picked)} / {len(rows)} -> {out}") + + +if __name__ == "__main__": + main() diff --git a/src/agent-harness/index.ts b/src/agent-harness/index.ts index a2666bd..40e843b 100644 --- a/src/agent-harness/index.ts +++ b/src/agent-harness/index.ts @@ -1,5 +1,6 @@ export * from "./image-highlights" export * from "./ledger" export * from "./knowhere-text" +export * from "./memory-text" export * from "./runtime" export * from "./types" diff --git a/src/agent-harness/memory-text.test.ts b/src/agent-harness/memory-text.test.ts new file mode 100644 index 0000000..aa1699d --- /dev/null +++ b/src/agent-harness/memory-text.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest" + +import { memoryToolText } from "./memory-text" + +describe("memoryToolText", () => { + it("formats search results with memory refs and stored summaries", () => { + const text = memoryToolText.formatSearch({ + query: "毛利率", + items: [ + { + ref: "mem:1", + itemId: "item_1", + kind: "stance", + abstractL0: "关注毛利率下滑", + overviewL1: "用户把毛利率当作核心观察指标。", + }, + ], + }) + + expect(text).toContain('') + expect(text).toContain('query="毛利率"') + expect(text).toContain('ref="mem:1"') + expect(text).toContain('itemId="item_1"') + expect(text).toContain('kind="stance"') + expect(text).toContain("关注毛利率下滑") + expect(text).toContain("用户把毛利率当作核心观察指标。") + }) + + it("formats an empty search without inventing items", () => { + const text = memoryToolText.formatSearch({ + query: "unknown", + items: [], + }) + + expect(text).toContain('resultCount="0"') + expect(text).not.toContain("", + ].join("\n") + }, +} as const + +function formatMemoryItems(response: MemorySearchResponse): string { + if (response.items.length === 0) return "" + + return [ + "", + ...response.items.map((item) => + [ + formatOpenTag("item", { + ref: item.ref, + itemId: item.itemId, + kind: item.kind, + }), + formatTextTag("abstract_l0", item.abstractL0), + formatTextTag("overview_l1", item.overviewL1), + "", + ].join("\n"), + ), + "", + ].join("\n") +} + +function wrapMemoryBlock( + operation: MemoryOperation, + parts: readonly string[], +): string { + return [ + formatOpenTag("memory", { operation, status: "ok" }), + ...parts.filter((part) => part.trim().length > 0), + "", + ].join("\n") +} + +function formatTextTag(tagName: string, value: string): string { + return [`<${tagName}>`, value, ``].join("\n") +} + +function formatTag( + tagName: string, + attrs: Readonly>, +): string { + return `${formatOpenTag(tagName, attrs)}` +} + +function formatOpenTag( + tagName: string, + attrs: Readonly>, +): string { + const serializedAttrs = Object.entries(attrs) + .filter((entry): entry is [string, string] => typeof entry[1] === "string") + .map(([key, value]) => `${key}="${escapeAttribute(value)}"`) + .join(" ") + return serializedAttrs ? `<${tagName} ${serializedAttrs}>` : `<${tagName}>` +} + +function escapeAttribute(value: string): string { + return value + .replaceAll("&", "&") + .replaceAll('"', """) + .replaceAll("<", "<") + .replaceAll(">", ">") +} diff --git a/src/agent-harness/runtime.test.ts b/src/agent-harness/runtime.test.ts index 74e7ef7..c2066c3 100644 --- a/src/agent-harness/runtime.test.ts +++ b/src/agent-harness/runtime.test.ts @@ -16,10 +16,23 @@ import type { ImageInspectionRequest, IntentFrame, KnowhereToolRuntime, + MemoryToolRuntime, OutputManifest, } from "./types" describe("agent harness runtime", () => { + it("tells the agent to search fluid memory first and not treat every question as document retrieval", () => { + const prompt = buildHarnessSystemPrompt(makeTurnInput()) + + expect(prompt).toContain("Call memory_search first") + expect(prompt).toContain( + "Call knowhere_search only when memory is insufficient and groundingPolicy requires citing source documents", + ) + expect(prompt).toContain( + "Do not treat every question as a document-retrieval task", + ) + }) + it("keeps KNOWHERE as an evidence provider instead of exposing internal navigation", () => { const prompt = buildHarnessSystemPrompt(makeTurnInput()) @@ -72,6 +85,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(query), recentTurns: [], }) @@ -147,6 +161,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(query), recentTurns: [], }) @@ -171,6 +186,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -203,6 +219,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -244,6 +261,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -274,6 +292,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -330,6 +349,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -435,6 +455,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -451,6 +472,7 @@ describe("agent harness runtime", () => { await executeTool(tools.finalize, { text: "Revenue was $24.9B [[cite:1]] [[cite:2]].", citations: [{ ref: "r1:result:1" }, { ref: "r1:result:2" }], + memoryCitations: [], artifacts: [], unresolved: [], }), @@ -523,6 +545,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -547,6 +570,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages: vi.fn().mockResolvedValue({ analysis: "", @@ -575,6 +599,7 @@ describe("agent harness runtime", () => { const finalize = await executeTool(tools.finalize, { text: "The amount is 5000 yuan [[cite:1]].", citations: [{ ref: "r1:referenced:1" }], + memoryCitations: [], artifacts: [], unresolved: [], }) @@ -593,6 +618,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), recentTurns: [], }) @@ -600,6 +626,7 @@ describe("agent harness runtime", () => { const manifest = { text: "Answer.", citations: [], + memoryCitations: [], artifacts: [], unresolved: [], } @@ -612,6 +639,60 @@ describe("agent harness runtime", () => { expect(state.finalized).toBe(true) }) + it("returns memory refs from memory_search and stores memoryCitations on finalize", async () => { + const search = vi.fn().mockResolvedValue({ + query: "毛利率", + items: [ + { + ref: "mem:1", + itemId: "item_1", + kind: "stance", + abstractL0: "关注毛利率下滑", + overviewL1: "用户把毛利率当作核心观察指标。", + }, + ], + }) + const state: { + finalizedManifest?: OutputManifest + finalized?: boolean + memorySearchInvoked?: boolean + } = {} + const tools = createHarnessTools({ + state, + ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(search), + knowhereTools: makeKnowhereTools(), + recentTurns: [], + }) + + const searchText = await executeTool(tools.memory_search, { + query: "毛利率", + }) + expect(searchText).toContain('') + expect(searchText).toContain('ref="mem:1"') + expect(searchText).toContain('itemId="item_1"') + expect(state.memorySearchInvoked).toBe(true) + expect(search).toHaveBeenCalledWith({ + query: "毛利率", + kinds: undefined, + }) + + const manifest = { + text: "按已有记忆，毛利率是核心观察指标。", + citations: [], + memoryCitations: [ + { ref: "mem:1", itemId: "item_1", kind: "stance" as const }, + ], + artifacts: [], + unresolved: [], + } + expect(await executeTool(tools.finalize, manifest)).toMatchObject({ + ok: true, + memoryCitations: manifest.memoryCitations, + }) + expect(state.finalizedManifest).toEqual(manifest) + }) + it("rejects finalize of cited page images until inspectImage has run", async () => { const ledger = createEvidenceLedger() ledger.addRetrievalResponse(makePageCitationRetrievalResponse()) @@ -623,6 +704,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages: vi.fn().mockResolvedValue({ analysis: "The clause shows 5000 yuan per occurrence.", @@ -640,6 +722,7 @@ describe("agent harness runtime", () => { const manifest = { text: "The contractor pays 5000 yuan per occurrence [[cite:1]].", citations: [{ ref: "r1:referenced:1" }], + memoryCitations: [], artifacts: [], unresolved: [], } @@ -692,6 +775,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages: vi.fn(), recentTurns: [], @@ -700,6 +784,7 @@ describe("agent harness runtime", () => { const result = await executeTool(tools.finalize, { text: "The contractor pays 5000 yuan [[cite:1]].", citations: [{ ref: "r1:result:1" }], + memoryCitations: [], artifacts: [], unresolved: [], }) @@ -725,6 +810,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), recentTurns: [ { @@ -757,6 +843,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), recentTurns: [ { @@ -881,7 +968,7 @@ describe("agent harness runtime", () => { ]) }) - it("keeps normal steps unconstrained before the finalization step", () => { + it("keeps retrieval tools closed until declareIntent allows them", () => { const result = prepareHarnessStep({ stepNumber: 11, messages: [ @@ -892,14 +979,93 @@ describe("agent harness runtime", () => { ], }) - expect(result).toEqual({ - messages: [ - { - role: "user", - content: "Find the penalty amount.", - }, - ], + expect(result.activeTools).toEqual([ + "declareIntent", + "setContextPolicy", + "inspectImage", + "readPriorTurn", + "finalize", + ]) + expect(result.activeTools).not.toContain("memory_search") + expect(result.activeTools).not.toContain("knowhere_search") + }) + + it("opens only memory_search after intent says retrieval may be needed", () => { + const result = prepareHarnessStep({ + stepNumber: 3, + intent: { + task: "answer", + dependsOnPreviousTurn: false, + retrievalNeeded: "maybe", + targetModalities: ["text"], + constraints: {}, + groundingPolicy: "can_use_context", + }, + messages: [], }) + + expect(result.activeTools).toContain("memory_search") + expect(result.activeTools).not.toContain("knowhere_search") + }) + + it("keeps Knowhere tools closed for no_retrieval even after memory_search", () => { + const result = prepareHarnessStep({ + stepNumber: 4, + memorySearchInvoked: true, + intent: { + task: "answer", + dependsOnPreviousTurn: false, + retrievalNeeded: "no", + targetModalities: ["text"], + constraints: {}, + groundingPolicy: "no_retrieval", + }, + messages: [], + }) + + expect(result.activeTools).not.toContain("memory_search") + expect(result.activeTools).not.toContain("knowhere_search") + }) + + it("opens Knowhere tools only after memory_search when sources are required", () => { + const beforeMemory = prepareHarnessStep({ + stepNumber: 3, + intent: { + task: "answer", + dependsOnPreviousTurn: false, + retrievalNeeded: "yes", + targetModalities: ["text"], + constraints: {}, + groundingPolicy: "must_use_sources", + }, + messages: [], + }) + const afterMemory = prepareHarnessStep({ + stepNumber: 4, + memorySearchInvoked: true, + intent: { + task: "answer", + dependsOnPreviousTurn: false, + retrievalNeeded: "yes", + targetModalities: ["text"], + constraints: {}, + groundingPolicy: "must_use_sources", + }, + messages: [], + }) + + expect(beforeMemory.activeTools).toContain("memory_search") + expect(beforeMemory.activeTools).not.toContain("knowhere_search") + expect(afterMemory.activeTools).toEqual( + expect.arrayContaining([ + "memory_search", + "knowhere_search", + "knowhere_list_documents", + "knowhere_get_document_outline", + "knowhere_read_chunks", + "knowhere_grep_chunks", + ]), + ) }) it("forces image inspection before forced finalization when image assets are available", () => { @@ -1034,6 +1200,14 @@ function executeTool(tool: unknown, input: unknown): Promise { return (tool as { execute: (input: unknown) => Promise }).execute(input) } +function makeMemoryTools( + search: MemoryToolRuntime["search"] = vi + .fn() + .mockResolvedValue({ query: "", items: [] }), +): MemoryToolRuntime { + return { search } +} + function makeKnowhereTools( search: KnowhereToolRuntime["search"] = vi.fn(), ): KnowhereToolRuntime { diff --git a/src/agent-harness/runtime.ts b/src/agent-harness/runtime.ts index 073051d..11a6c3a 100644 --- a/src/agent-harness/runtime.ts +++ b/src/agent-harness/runtime.ts @@ -10,6 +10,7 @@ import { z } from "zod" import { createEvidenceLedger } from "./ledger" import { getCanonicalImageAssetKey } from "./image-asset-identity" import { knowhereToolText } from "./knowhere-text" +import { memoryToolText } from "./memory-text" import { mergeImageInspectionHighlights } from "./image-highlights" import type { AgentTurn, @@ -27,8 +28,11 @@ import type { IntentFrame, KnowhereSearchTargetContent, KnowhereToolRuntime, + MemorySearchKind, + MemoryToolRuntime, OutputManifest, } from "./types" +import { memorySearchKinds } from "./types" const defaultMaxSteps = 14 const imageInspectionReminderStepNumber = 12 @@ -42,6 +46,7 @@ export type RunAgentHarnessInput = { readonly model: AgentHarnessModel readonly turn: AgentTurnInput readonly knowhereTools: KnowhereToolRuntime + readonly memoryTools: MemoryToolRuntime readonly inspectImages?: InspectImages readonly maxSteps?: number } @@ -55,6 +60,7 @@ type HarnessToolState = { inspectedImageRefs?: string[] imageHighlights?: ImageInspectionHighlights[] toolCalls?: HarnessToolCallTrace[] + memorySearchInvoked?: boolean } type HarnessTools = ReturnType @@ -171,6 +177,17 @@ const outputCitationSchema = z.object({ .optional(), }) +const memoryCitationSchema = z.object({ + ref: z.string().min(1), + itemId: z.string().min(1), + kind: z.enum(memorySearchKinds), +}) + +const memorySearchSchema = z.object({ + query: z.string().min(1), + kinds: z.array(z.enum(memorySearchKinds)).optional(), +}) + const selectedOutputArtifactSchema = z.object({ type: z.enum(["image", "table"]), ref: z.string().min(1), @@ -197,6 +214,7 @@ const outputArtifactSchema = z.union([ const outputManifestSchema = z.object({ text: z.string(), citations: z.array(outputCitationSchema).default([]), + memoryCitations: z.array(memoryCitationSchema).default([]), artifacts: z.array(outputArtifactSchema).default([]), unresolved: z.array(z.string()).default([]), }) @@ -214,6 +232,7 @@ export async function runAgentHarness( state, ledger, knowhereTools: input.knowhereTools, + memoryTools: input.memoryTools, inspectImages: input.inspectImages, recentTurns: input.turn.recentTurns, }) @@ -225,6 +244,8 @@ export async function runAgentHarness( prepareHarnessStep({ messages: stepMessages, stepNumber, + intent: state.intent, + memorySearchInvoked: state.memorySearchInvoked === true, hasUninspectedImageAssets: input.inspectImages !== undefined && hasUninspectedImageAssets({ state, ledger }), @@ -257,10 +278,33 @@ export async function runAgentHarness( } } +const alwaysAvailableTools = [ + "declareIntent", + "setContextPolicy", + "inspectImage", + "readPriorTurn", + "finalize", +] as const + +const fluidRetrievalTools = ["memory_search"] as const + +const crystalRetrievalTools = [ + "knowhere_search", + "knowhere_list_documents", + "knowhere_get_document_outline", + "knowhere_read_chunks", + "knowhere_grep_chunks", +] as const + +/** Reserved third retrieval slot (cognition). Not registered this round. */ +const cognitionRetrievalTools = [] as const + export function prepareHarnessStep(input: { readonly stepNumber: number readonly messages: readonly ModelMessage[] readonly hasUninspectedImageAssets?: boolean + readonly intent?: IntentFrame + readonly memorySearchInvoked?: boolean }): HarnessStepPreparation { const messages = sanitizeHarnessModelMessagesForStep(input.messages) @@ -285,24 +329,59 @@ export function prepareHarnessStep(input: { } } - if (input.stepNumber < forcedFinalizationStepNumber) { - return { messages } + if (input.stepNumber >= forcedFinalizationStepNumber) { + return { + messages: [ + ...messages, + { + role: "user", + content: buildForcedFinalizationFeedback(), + }, + ], + activeTools: ["finalize"], + toolChoice: { + type: "tool", + toolName: "finalize", + }, + } } return { - messages: [ - ...messages, - { - role: "user", - content: buildForcedFinalizationFeedback(), - }, - ], - activeTools: ["finalize"], - toolChoice: { - type: "tool", - toolName: "finalize", - }, + messages, + activeTools: selectHarnessActiveTools({ + intent: input.intent, + memorySearchInvoked: input.memorySearchInvoked === true, + }), + } +} + +function selectHarnessActiveTools(input: { + readonly intent?: IntentFrame + readonly memorySearchInvoked: boolean +}): Array> { + const tools: Array> = [ + ...alwaysAvailableTools, + ] + if (!allowsRetrieval(input.intent)) { + return tools } + + tools.push(...fluidRetrievalTools) + if ( + input.intent?.groundingPolicy === "must_use_sources" && + input.memorySearchInvoked + ) { + tools.push(...crystalRetrievalTools) + } + tools.push(...cognitionRetrievalTools) + return tools +} + +function allowsRetrieval(intent?: IntentFrame): boolean { + if (!intent) return false + return ( + intent.groundingPolicy !== "no_retrieval" && intent.retrievalNeeded !== "no" + ) } export function sanitizeHarnessModelMessagesForStep( @@ -410,6 +489,7 @@ export function createHarnessTools(input: { readonly state: HarnessToolState readonly ledger: ReturnType readonly knowhereTools: KnowhereToolRuntime + readonly memoryTools: MemoryToolRuntime readonly inspectImages?: InspectImages readonly recentTurns: readonly AgentTurn[] }) { @@ -446,6 +526,26 @@ export function createHarnessTools(input: { }), }), + memory_search: tool({ + description: + "Search distilled fluid memory for this workspace. Returns tagged text with memory refs such as mem:1. Use this before Knowhere document search.", + inputSchema: memorySearchSchema, + execute: async (request) => + traceToolCall(input.state, { + toolName: "memory_search", + inputSummary: summarizeMemorySearchRequest(request), + execute: async () => { + const output = await executeMemorySearch({ + memoryTools: input.memoryTools, + request, + }) + input.state.memorySearchInvoked = true + return output + }, + summarizeOutput: summarizeMemoryTextOutput, + }), + }), + knowhere_search: tool({ description: "Search Knowhere for relevant Notebook evidence. Returns tagged text with evidence refs such as r1:result:1 and asset refs such as asset:r1:result:1.", @@ -628,6 +728,7 @@ export function createHarnessTools(input: { "Finalize the user-facing output manifest. This is the only final answer " + "contract. Artifacts listed here with display=true are the exact set of " + "images/tables shown to the user; cite evidence refs when available. " + + "Use citations for Knowhere evidence and memoryCitations for fluid memory refs. " + "Cited page/image assets must be inspected with inspectImage first.", inputSchema: outputManifestSchema, execute: async (manifest) => @@ -912,6 +1013,7 @@ type KnowhereToolOperation = | "read_chunks" | "grep_chunks" +type MemorySearchToolRequest = z.infer type KnowhereSearchToolRequest = z.infer type KnowhereDocumentReferenceRequest = z.infer< typeof knowhereDocumentReferenceSchema @@ -925,6 +1027,24 @@ type DocumentReferenceSummary = { readonly hasRevisionKey: boolean } +async function executeMemorySearch(input: { + readonly memoryTools: MemoryToolRuntime + readonly request: MemorySearchToolRequest +}): Promise { + try { + const response = await input.memoryTools.search({ + query: input.request.query, + kinds: input.request.kinds, + }) + return memoryToolText.formatSearch(response) + } catch (error) { + return memoryToolText.formatError({ + operation: "search", + message: formatUnknownError(error), + }) + } +} + async function executeKnowhereSearch(input: { readonly ledger: ReturnType readonly knowhereTools: KnowhereToolRuntime @@ -1114,6 +1234,25 @@ function summarizeContextPolicy(policy: ContextPolicy): unknown { } } +function summarizeMemorySearchRequest(request: { + readonly query: string + readonly kinds?: readonly MemorySearchKind[] +}): unknown { + return { + query: request.query, + kinds: request.kinds, + } +} + +function summarizeMemoryTextOutput(output: unknown): unknown { + if (typeof output !== "string") return output + return { + ok: !output.includes('status="error"'), + textLength: output.length, + itemCount: countOccurrences(output, " artifact.display) .length, @@ -1255,6 +1395,9 @@ function summarizeFinalizeOutput(output: unknown): unknown { ok: output.ok, textLength: typeof output.text === "string" ? output.text.length : 0, citationCount: Array.isArray(output.citations) ? output.citations.length : 0, + memoryCitationCount: Array.isArray(output.memoryCitations) + ? output.memoryCitations.length + : 0, artifactCount: Array.isArray(output.artifacts) ? output.artifacts.length : 0, unresolvedCount: Array.isArray(output.unresolved) ? output.unresolved.length @@ -1282,12 +1425,17 @@ export function buildHarnessSystemPrompt(turn: AgentTurnInput): string { "1. Call declareIntent when it helps you plan the response. Capture constraints like a requested image/table count in constraints.desiredCount.", "2. Call setContextPolicy when prior turns may influence this turn.", "3. When the policy needs prior-turn detail (references or corrections), call readPriorTurn for the relevant ids.", - "4. Call knowhere_search when relevance search is needed. Use knowhere_list_documents, knowhere_get_document_outline, knowhere_read_chunks, and knowhere_grep_chunks for focused document reads.", + "4. Call memory_search first when known fluid memory may answer the request. Call knowhere_search only when memory is insufficient and groundingPolicy requires citing source documents. Use knowhere_list_documents, knowhere_get_document_outline, knowhere_read_chunks, and knowhere_grep_chunks for focused document reads.", "5. After Knowhere returns image/page asset refs, call inspectImage on the page/image assets you will cite before finalize. This supplies OCR/visual context and provenance boxes.", "6. Inspect each unique cited page once; retrieval already bounds the available evidence set.", "7. knowhere_read_chunks returns complete chunk bodies; control size with page/pageSize, sectionPath, startChunk/endChunk, chunkId, and chunkType.", "8. Call finalize with text, citations, artifacts, and unresolved issues when you are ready to answer.", "", + "Retrieval rules:", + "- First use memory_search to see whether known fluid memory can answer directly.", + "- Call knowhere_search only when memory is insufficient and groundingPolicy requires citing source documents.", + "- Do not treat every question as a document-retrieval task.", + "", "Context rules:", "- If the current user request is unrelated to prior turns, set carryHistory to none and do not reuse prior topics.", "- If the user corrects a previous answer, set carryHistory to repair_previous, read the relevant prior turn, then re-retrieve and re-answer using the correction.", @@ -1347,6 +1495,7 @@ function buildFallbackManifest(text: string): OutputManifest { return { text, citations: [], + memoryCitations: [], artifacts: [], unresolved: text ? [] : ["The agent did not finalize an output manifest."], } diff --git a/src/agent-harness/types.ts b/src/agent-harness/types.ts index edd9c2f..82f38ab 100644 --- a/src/agent-harness/types.ts +++ b/src/agent-harness/types.ts @@ -131,6 +131,43 @@ export type KnowhereToolRuntime = { ) => Promise } +export const memorySearchKinds = [ + "indicator_pref", + "stance", + "decision_rule", + "entity_of_interest", +] as const + +export type MemorySearchKind = (typeof memorySearchKinds)[number] + +export type MemorySearchRequest = { + readonly query: string + readonly kinds?: readonly MemorySearchKind[] +} + +export type MemorySearchItem = { + readonly ref: string + readonly itemId: string + readonly kind: MemorySearchKind + readonly abstractL0: string + readonly overviewL1: string +} + +export type MemorySearchResponse = { + readonly query: string + readonly items: readonly MemorySearchItem[] +} + +export type MemoryToolRuntime = { + readonly search: (input: MemorySearchRequest) => Promise +} + +export type MemoryCitation = { + readonly ref: string + readonly itemId: string + readonly kind: MemorySearchKind +} + export type EvidenceChunk = { readonly ref: string readonly kind: "result" | "referenced_chunk" | "read_chunk" | "grep_match" @@ -254,6 +291,7 @@ export type OutputArtifactView = OutputArtifact | DerivedTableArtifact export type OutputManifest = { readonly text: string readonly citations: readonly OutputCitation[] + readonly memoryCitations: readonly MemoryCitation[] readonly artifacts: readonly OutputArtifactView[] readonly unresolved: readonly string[] } diff --git a/src/domains/chat/citations.test.ts b/src/domains/chat/citations.test.ts index b381a83..1335de3 100644 --- a/src/domains/chat/citations.test.ts +++ b/src/domains/chat/citations.test.ts @@ -64,6 +64,21 @@ describe("toChatCitationViews", () => { expect(citations[0]?.pageCitationPageNumber).toBe(26) }) + it("copies the chunk id onto the citation when the retrieval result has one", () => { + const citations = toChatCitationViews( + [makeRetrievalResult({ chunkId: "chunk_123" })], + "Grounded answer.", + ) + + expect(citations[0]?.chunkId).toBe("chunk_123") + }) + + it("omits chunkId when the retrieval result has none", () => { + const citations = toChatCitationViews([makeRetrievalResult()], "Grounded answer.") + + expect(citations[0]).not.toHaveProperty("chunkId") + }) + it("copies inspect-image provenance boxes onto the citation", () => { const citations = toChatCitationViews( [ diff --git a/src/domains/chat/citations.ts b/src/domains/chat/citations.ts index 9fb6c8d..18304aa 100644 --- a/src/domains/chat/citations.ts +++ b/src/domains/chat/citations.ts @@ -20,6 +20,7 @@ export function toChatCitationViews( content: result.content, chunkType: result.chunkType, score: result.score, + ...(result.chunkId ? { chunkId: result.chunkId } : {}), ...(result.assetUrl ? { assetUrl: result.assetUrl } : {}), ...(result.pageCitationAssetUrl ? { pageCitationAssetUrl: result.pageCitationAssetUrl } diff --git a/src/domains/chat/commit-turn.test.ts b/src/domains/chat/commit-turn.test.ts new file mode 100644 index 0000000..62e20b3 --- /dev/null +++ b/src/domains/chat/commit-turn.test.ts @@ -0,0 +1,138 @@ +import { Either } from "effect" +import { beforeEach, describe, expect, it, vi } from "vitest" + +const mocks = vi.hoisted(() => ({ + handleChatTurn: vi.fn(), + triggerMemoryExtraction: vi.fn(), + recordActivations: vi.fn(), +})) + +vi.mock("./service", () => ({ + handleChatTurn: mocks.handleChatTurn, +})) + +vi.mock("@/domains/memory/extract-trigger", () => ({ + triggerMemoryExtraction: mocks.triggerMemoryExtraction, +})) + +vi.mock("@/domains/retrieval-activation/service", () => ({ + retrievalActivationService: { + recordActivations: mocks.recordActivations, + }, +})) + +import { commitChatTurn } from "./commit-turn" +import type { Workspace } from "@/infrastructure/db/schema" + +describe("commitChatTurn", () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.recordActivations.mockResolvedValue(1) + }) + + it("records fluid memory activations from finalize after a successful turn", async () => { + mocks.handleChatTurn.mockImplementation(async (input) => { + await input.generateAnswer({ + question: "毛利率", + messages: [], + sources: [], + excludedSourceIds: [], + searchSources: vi.fn(), + }) + return Either.right({ + threadId: "thread_1", + messages: [ + { id: "msg_user", role: "user", content: "毛利率" }, + { + id: "msg_assistant", + role: "assistant", + content: "按已有记忆。", + citations: [ + { + chunkId: "chunk_1", + chunkType: "text", + score: 0.9, + source: { documentId: "doc_1" }, + }, + ], + }, + ], + }) + }) + + const generateAnswer = vi.fn(async () => ({ + manifest: { + text: "按已有记忆。", + citations: [], + memoryCitations: [ + { ref: "mem:1", itemId: "item_1", kind: "stance" as const }, + ], + artifacts: [], + unresolved: [], + }, + trace: { + ledger: { + retrievalCount: 0, + chunks: [], + assets: [], + evidenceText: [], + stopReasons: [], + failureReasons: [], + decisionTraces: [], + }, + finalized: true, + priorTurnReads: [], + toolCalls: [], + imageHighlights: [], + validationErrors: [], + revisionsUsed: 0, + }, + })) + + const result = await commitChatTurn({ + workspace: makeWorkspace(), + sources: [], + question: "毛利率", + excludedSourceIds: [], + retrieval: { query: vi.fn() }, + generateAnswer, + repository: { + ensureDefaultChatThread: vi.fn(), + findChatThreadInWorkspace: vi.fn(), + listMessagesForThread: vi.fn(), + appendMessageToThread: vi.fn(), + }, + }) + + expect(Either.isRight(result)).toBe(true) + expect(mocks.triggerMemoryExtraction).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + threadId: "thread_1", + userMessageId: "msg_user", + assistantMessageId: "msg_assistant", + }) + expect(mocks.recordActivations).toHaveBeenCalledWith([ + { + workspaceId: "workspace_1", + unitType: "crystal_chunk", + unitRef: "doc_1:chunk_1", + }, + ]) + expect(mocks.recordActivations).toHaveBeenCalledWith([ + { + workspaceId: "workspace_1", + unitType: "fluid_memory", + unitRef: "item_1", + }, + ]) + }) +}) + +function makeWorkspace(): Workspace { + return { + id: "workspace_1", + userId: "user_1", + namespace: "notebook-namespace", + createdAt: new Date("2026-09-10T00:00:00Z"), + } +} diff --git a/src/domains/chat/commit-turn.ts b/src/domains/chat/commit-turn.ts new file mode 100644 index 0000000..373bb3a --- /dev/null +++ b/src/domains/chat/commit-turn.ts @@ -0,0 +1,131 @@ +import { Either } from "effect" + +import type { MemoryCitation } from "@/agent-harness" +import { generateAgenticOutputManifest } from "./prompt" +import { triggerMemoryExtraction } from "@/domains/memory/extract-trigger" +import { retrievalActivationService } from "@/domains/retrieval-activation/service" +import { toChunkUnitRef } from "@/domains/retrieval-activation/types" +import { summarizeUnknownError } from "@/lib/format-log-value" +import { logger } from "@/lib/logger" +import type { ChatCitationView } from "./types" +import { + handleChatTurn, + type ChatTurnError, + type ChatTurnValue, +} from "./service" + +type CommitChatTurnInput = Parameters[0] + +/** + * Production chat-turn commit used by the HTTP route and the 观心 batch + * script: answer → persist → extract fluid memory → record cited + * crystal/memory activations. + */ +export async function commitChatTurn( + input: CommitChatTurnInput, +): Promise> { + let memoryCitations: readonly MemoryCitation[] = [] + const result = await handleChatTurn({ + ...input, + generateAnswer: async (generateInput) => { + const generated = await input.generateAnswer({ + ...generateInput, + }) + memoryCitations = generated.manifest.memoryCitations + return generated + }, + }) + + if (Either.isRight(result)) { + void triggerMemoryExtraction({ + workspaceId: input.workspace.id, + threadId: result.right.threadId, + userMessageId: result.right.messages[0].id, + assistantMessageId: result.right.messages[1].id, + }) + void recordChunkActivations({ + workspaceId: input.workspace.id, + citations: result.right.messages[1].citations, + }) + void recordMemoryActivations({ + workspaceId: input.workspace.id, + memoryCitations, + }) + } + + return result +} + +export async function commitAgenticChatTurn( + input: Omit, +): Promise> { + return commitChatTurn({ + ...input, + generateAnswer: (generateInput) => + generateAgenticOutputManifest({ + ...generateInput, + workspaceId: input.workspace.id, + }), + }) +} + +/** + * Fire-and-forget activation ledger write for crystal chunks actually cited + * in this turn's answer. Only citations with both a documentId and a + * chunkId count — a citation missing either can't be identified down to a + * chunk (see ChatCitationView / RetrievalResultView). + */ +export async function recordChunkActivations(input: { + readonly workspaceId: string + readonly citations: readonly ChatCitationView[] | undefined +}): Promise { + const activationInputs = (input.citations ?? []).flatMap((citation) => { + const documentId = citation.source.documentId + const chunkId = citation.chunkId + if (!documentId || !chunkId) return [] + return [ + { + workspaceId: input.workspaceId, + unitType: "crystal_chunk" as const, + unitRef: toChunkUnitRef({ documentId, chunkId }), + }, + ] + }) + if (activationInputs.length === 0) return + + try { + await retrievalActivationService.recordActivations(activationInputs) + } catch (error) { + logger.warn("chat: failed to record chunk activations", { + workspaceId: input.workspaceId, + chunkCount: activationInputs.length, + error: summarizeUnknownError(error), + }) + } +} + +/** + * Fire-and-forget activation ledger write for fluid memory items actually + * cited in this turn's finalize output. + */ +export async function recordMemoryActivations(input: { + readonly workspaceId: string + readonly memoryCitations: readonly MemoryCitation[] +}): Promise { + const activationInputs = input.memoryCitations.map((citation) => ({ + workspaceId: input.workspaceId, + unitType: "fluid_memory" as const, + unitRef: citation.itemId, + })) + if (activationInputs.length === 0) return + + try { + await retrievalActivationService.recordActivations(activationInputs) + } catch (error) { + logger.warn("chat: failed to record memory activations", { + workspaceId: input.workspaceId, + memoryCount: activationInputs.length, + error: summarizeUnknownError(error), + }) + } +} diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index 1bc9b60..021ebb2 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -1040,6 +1040,7 @@ describe("answerQuestionWithRetrieval", () => { manifest: { text: `Use this image. ${rawAssetUrl}`, citations: [], + memoryCitations: [], artifacts: [ { type: "image", @@ -1773,6 +1774,7 @@ describe("answerQuestionWithRetrieval", () => { manifest: { text: "已找到相关身份证图片，见下方图片。", citations: [], + memoryCitations: [], artifacts: [ { type: "image", @@ -2063,6 +2065,7 @@ describe("answerQuestionWithRetrieval", () => { }, }, ], + memoryCitations: [], artifacts: [], unresolved: [], }); @@ -2081,7 +2084,11 @@ describe("answerQuestionWithRetrieval", () => { sources: [makeSource()], excludedSourceIds: [], retrieval, - generateAnswer: generateAgenticOutputManifest, + generateAnswer: (input) => + generateAgenticOutputManifest({ + ...input, + workspaceId: "workspace_1", + }), messages: [], }), ); @@ -2128,6 +2135,7 @@ describe("answerQuestionWithRetrieval", () => { manifest: { text: "", citations: [], + memoryCitations: [], artifacts: [ { type: "image", @@ -2243,6 +2251,7 @@ describe("answerQuestionWithRetrieval", () => { manifest: { text: "I organized the comparison into a table.", citations: [], + memoryCitations: [], artifacts: [ { type: "derived_table", @@ -2639,6 +2648,7 @@ describe("answerQuestionWithRetrieval", () => { content: "", chunkType: "image", score: null, + chunkId: "chunk_1", assetUrl: "https://blob.example/images/launch.jpg", source: { documentId: "doc_spacex", @@ -2699,6 +2709,7 @@ describe("generateAgenticOutputManifest", () => { }, }, ], + memoryCitations: [], artifacts: [ { type: "image", @@ -2739,6 +2750,7 @@ describe("generateAgenticOutputManifest", () => { }); const result = await generateAgenticOutputManifest({ + workspaceId: "workspace_1", question: "请只返回冯荣洲的 2 张身份证图片", messages: [ { @@ -2836,6 +2848,7 @@ describe("generateAgenticOutputManifest", () => { }, }, ], + memoryCitations: [], artifacts: [ { type: "image", @@ -2885,6 +2898,7 @@ describe("generateAgenticOutputManifest", () => { }); const result = await generateAgenticOutputManifest({ + workspaceId: "workspace_1", question: "Inspect and show the ID card image.", messages: [], sources: [ @@ -2976,6 +2990,7 @@ describe("generateAgenticOutputManifest", () => { }, }, ], + memoryCitations: [], artifacts: [], unresolved: [], }); @@ -3029,6 +3044,7 @@ describe("generateAgenticOutputManifest", () => { }); const result = await generateAgenticOutputManifest({ + workspaceId: "workspace_1", question: "承包人自行修改发包人审批的进度时需要赔偿多少违约金？", messages: [], sources: [ @@ -3126,6 +3142,7 @@ describe("generateAgenticOutputManifest", () => { }, }, ], + memoryCitations: [], artifacts: [1, 2, 3].map((index) => ({ type: "image", ref: `asset:r1:result:${index}`, @@ -3148,6 +3165,7 @@ describe("generateAgenticOutputManifest", () => { }, }, ], + memoryCitations: [], artifacts: [1, 2].map((index) => ({ type: "image", ref: `asset:r1:result:${index}`, @@ -3188,6 +3206,7 @@ describe("generateAgenticOutputManifest", () => { }); const result = await generateAgenticOutputManifest({ + workspaceId: "workspace_1", question: "只要 2 张身份证图片", messages: [], sources: [ @@ -3322,6 +3341,7 @@ function makeHarnessRunResult(text: string): HarnessRunResult { manifest: { text, citations: [], + memoryCitations: [], artifacts: [], unresolved: [], }, @@ -3371,6 +3391,7 @@ function makeHarnessRunResultWithLedger( manifest: { text, citations: input.citations ?? [], + memoryCitations: [], artifacts: input.artifacts ?? [], unresolved: [], }, diff --git a/src/domains/chat/memory-tools.test.ts b/src/domains/chat/memory-tools.test.ts new file mode 100644 index 0000000..ad083a9 --- /dev/null +++ b/src/domains/chat/memory-tools.test.ts @@ -0,0 +1,104 @@ +import { beforeEach, describe, expect, it, vi } from "vitest" + +import { DISTILL_DEDUP_CANDIDATES_PER_KIND } from "@/domains/memory/distill-config" +import type { FluidMemoryItem } from "@/infrastructure/db/schema" + +const findDedupCandidates = vi.fn() + +vi.mock("@/domains/memory/service", () => ({ + memoryService: { + findDedupCandidates: (...args: unknown[]) => findDedupCandidates(...args), + }, +})) + +describe("notebookMemoryTools", () => { + beforeEach(() => { + findDedupCandidates.mockReset() + }) + + it("queries all four kinds and assigns mem refs in kind order", async () => { + findDedupCandidates.mockImplementation( + async (_workspaceId: string, kind: string) => { + if (kind === "stance") return [makeMemoryItem({ id: "item_stance" })] + if (kind === "entity_of_interest") { + return [makeMemoryItem({ id: "item_entity", kind: "entity_of_interest" })] + } + return [] + }, + ) + const { notebookMemoryTools } = await import("./memory-tools") + const runtime = notebookMemoryTools.createRuntime({ + workspaceId: "workspace_1", + }) + + const response = await runtime.search({ query: "毛利率 英伟达" }) + + expect(findDedupCandidates).toHaveBeenCalledTimes(4) + expect(findDedupCandidates.mock.calls.map((call) => call[1])).toEqual([ + "indicator_pref", + "stance", + "decision_rule", + "entity_of_interest", + ]) + expect(findDedupCandidates.mock.calls[0]?.[3]).toBe( + DISTILL_DEDUP_CANDIDATES_PER_KIND, + ) + expect(response).toEqual({ + query: "毛利率 英伟达", + items: [ + expect.objectContaining({ + ref: "mem:1", + itemId: "item_stance", + kind: "stance", + }), + expect.objectContaining({ + ref: "mem:2", + itemId: "item_entity", + kind: "entity_of_interest", + }), + ], + }) + }) + + it("searches only the requested kinds", async () => { + findDedupCandidates.mockResolvedValue([]) + const { notebookMemoryTools } = await import("./memory-tools") + const runtime = notebookMemoryTools.createRuntime({ + workspaceId: "workspace_1", + }) + + await runtime.search({ + query: "PE", + kinds: ["indicator_pref"], + }) + + expect(findDedupCandidates).toHaveBeenCalledTimes(1) + expect(findDedupCandidates).toHaveBeenCalledWith( + "workspace_1", + "indicator_pref", + expect.any(Array), + DISTILL_DEDUP_CANDIDATES_PER_KIND, + ) + }) +}) + +function makeMemoryItem( + overrides: Partial = {}, +): FluidMemoryItem { + return { + id: "item_1", + workspaceId: "workspace_1", + kind: "stance", + payload: { statement: "s", scope: "scope", rationale: "r" }, + abstractL0: "abstract", + overviewL1: "overview", + sourceMessageId: null, + confidence: 0.8, + status: "active", + deactivationReason: null, + version: 1, + createdAt: new Date("2026-09-10T00:00:00Z"), + updatedAt: new Date("2026-09-10T00:00:00Z"), + ...overrides, + } +} diff --git a/src/domains/chat/memory-tools.ts b/src/domains/chat/memory-tools.ts new file mode 100644 index 0000000..63d48b1 --- /dev/null +++ b/src/domains/chat/memory-tools.ts @@ -0,0 +1,66 @@ +import type { + MemorySearchItem, + MemorySearchRequest, + MemorySearchResponse, + MemoryToolRuntime, +} from "@/agent-harness" +import { DISTILL_DEDUP_CANDIDATES_PER_KIND } from "@/domains/memory/distill-config" +import { tokenizeMemoryText } from "@/domains/memory/search-index" +import { memoryService } from "@/domains/memory/service" +import { fluidMemoryKinds, isFluidMemoryKind } from "@/domains/memory/types" +import type { FluidMemoryItem } from "@/infrastructure/db/schema" + +type NotebookMemoryToolsInput = { + readonly workspaceId: string +} + +export const notebookMemoryTools = { + createRuntime(input: NotebookMemoryToolsInput): MemoryToolRuntime { + return { + search: (request) => searchWorkspaceMemory(input.workspaceId, request), + } + }, +} as const + +async function searchWorkspaceMemory( + workspaceId: string, + request: MemorySearchRequest, +): Promise { + const tokens = tokenizeMemoryText(request.query).map((entry) => entry.token) + const kinds = request.kinds ?? fluidMemoryKinds + const items: MemorySearchItem[] = [] + + for (const kind of kinds) { + const candidates = await memoryService.findDedupCandidates( + workspaceId, + kind, + tokens, + // Same per-kind cap as the existing findDedupCandidates caller. + DISTILL_DEDUP_CANDIDATES_PER_KIND, + ) + for (const candidate of candidates) { + const item = toMemorySearchItem(candidate, items.length + 1) + if (item) items.push(item) + } + } + + return { + query: request.query, + items, + } +} + +function toMemorySearchItem( + item: FluidMemoryItem, + index: number, +): MemorySearchItem | null { + if (!isFluidMemoryKind(item.kind)) return null + + return { + ref: `mem:${index}`, + itemId: item.id, + kind: item.kind, + abstractL0: item.abstractL0, + overviewL1: item.overviewL1, + } +} diff --git a/src/domains/chat/prompt.ts b/src/domains/chat/prompt.ts index a4ae326..d61177e 100644 --- a/src/domains/chat/prompt.ts +++ b/src/domains/chat/prompt.ts @@ -17,12 +17,14 @@ import type { SearchSources, } from "./contracts" import { notebookKnowhereTools } from "./knowhere-tools" +import { notebookMemoryTools } from "./memory-tools" const RECENT_CONTEXT_MESSAGE_LIMIT = 8 const CONTEXT_CONTENT_CHAR_LIMIT = 900 const SOURCE_CONTEXT_LIMIT = 12 type GenerateAgenticOutputManifestInput = { + workspaceId: string question: string messages: readonly ChatHistoryMessage[] sources: readonly Source[] @@ -63,6 +65,9 @@ export const generateAgenticOutputManifestEffect = ( notebookKnowhereTools.createSearchOnlyRuntime({ searchSources: input.searchSources, }), + memoryTools: notebookMemoryTools.createRuntime({ + workspaceId: input.workspaceId, + }), ...(input.inspectImages ? { inspectImages: input.inspectImages } : {}), }), ) diff --git a/src/domains/chat/route-answer.ts b/src/domains/chat/route-answer.ts index cc579b0..0a7cbb4 100644 --- a/src/domains/chat/route-answer.ts +++ b/src/domains/chat/route-answer.ts @@ -1,9 +1,6 @@ import { Cause, Effect, Either, Option } from "effect" -import { - generateAgenticOutputManifest, - parseChatRequestBody, -} from "@/domains/chat" +import { parseChatRequestBody } from "@/domains/chat" import type { ImageInspectionAsset, ImageInspectionRequest, @@ -11,16 +8,15 @@ import type { ImageInspectionSkippedAsset, InspectImages, } from "@/agent-harness" +import { commitAgenticChatTurn } from "@/domains/chat/commit-turn" import { normalizeImageInspectionHighlights } from "@/agent-harness/image-highlights" import { generateImageInspectionModelResult } from "@/domains/chat/image-inspection-model" import { hardenChatMediaAssetUrls } from "@/domains/chat/media-asset-hardening" import { - handleChatTurn, type ChatTurnError, type ChatTurnValue, } from "@/domains/chat/service" import { chatTurnPersistence } from "@/domains/chat/chat-turn-persistence" -import { triggerMemoryExtraction } from "@/domains/memory/extract-trigger" import { startBackgroundReconciliation } from "@/domains/sources/background-reconcile" import { BlobParsedDocumentStorage } from "@/domains/sources/parsed-document-blob-storage" import { sourceWorkflowRuntime } from "@/domains/sources/workflow-runtime" @@ -140,7 +136,7 @@ const answerChatEffect = (input: AnswerChatInput) => const result: Either.Either = yield* Effect.tryPromise(() => - handleChatTurn({ + commitAgenticChatTurn({ workspace, sources, question: body.value.question, @@ -150,7 +146,6 @@ const answerChatEffect = (input: AnswerChatInput) => retrieval: client.retrieval, knowledge: knowhereResources.knowledge, remoteDocumentClient: client, - generateAnswer: generateAgenticOutputManifest, hardenChatAssetUrl, hardenMediaAssetUrls: ({ results, artifacts }) => hardenChatMediaAssetUrls({ @@ -197,17 +192,8 @@ const answerChatEffect = (input: AnswerChatInput) => return Either.match(result, { onLeft: (error): RouteResponse => routeResult.error(error.status, error.message), - onRight: (value): RouteResponse => { - // Fire-and-forget: extract fluid memory from this turn without - // blocking the chat response. - void triggerMemoryExtraction({ - workspaceId: workspace.id, - threadId: value.threadId, - userMessageId: value.messages[0].id, - assistantMessageId: value.messages[1].id, - }) - return routeResult.ok(value) - }, + onRight: (value): RouteResponse => + routeResult.ok(value), }) }) diff --git a/src/domains/chat/route-service.test.ts b/src/domains/chat/route-service.test.ts index 02eceb9..54e8dae 100644 --- a/src/domains/chat/route-service.test.ts +++ b/src/domains/chat/route-service.test.ts @@ -191,7 +191,7 @@ describe("chat route services", () => { useAgentic: true, excludedSourceIds: ["source_skipped"], retrieval: client.retrieval, - generateAnswer: mocks.generateAgenticOutputManifest, + generateAnswer: expect.any(Function), hardenChatAssetUrl: expect.any(Function), repository: expect.objectContaining({ appendMessageToThread: expect.any(Function), diff --git a/src/domains/chat/service.test.ts b/src/domains/chat/service.test.ts index b065251..0ac2387 100644 --- a/src/domains/chat/service.test.ts +++ b/src/domains/chat/service.test.ts @@ -83,6 +83,39 @@ describe("handleChatTurn", () => { }); }); + it("allows a turn with no local sources so remote retrieval can still run", async () => { + const retrieval = { + query: vi.fn().mockResolvedValue({ + results: [makeRetrievalResult()], + evidenceText: "Grounding content", + referencedChunks: [], + namespace: "notebook-namespace", + query: "What does the document say?", + routerUsed: "workflow_single_step", + answerText: null, + }), + }; + const repository = makeRepository(); + const generateAnswer = vi.fn(async ({ searchSources }) => { + await searchSources({ query: "What does the document say?" }); + return makeHarnessRunResult("Grounded answer."); + }); + + const result = await handleChatTurn({ + workspace: makeWorkspace(), + sources: [], + question: "What does the document say?", + excludedSourceIds: [], + retrieval, + generateAnswer, + repository, + }); + + expect(Either.isRight(result)).toBe(true); + expect(generateAnswer).toHaveBeenCalled(); + expect(repository.appendMessageToThread).toHaveBeenCalled(); + }); + it("rejects chat before any source is ready without calling retrieval", async () => { const retrieval = { query: vi.fn() }; const repository = makeRepository(); @@ -334,6 +367,7 @@ function makeHarnessRunResult(text: string): HarnessRunResult { manifest: { text, citations: [], + memoryCitations: [], artifacts: [], unresolved: [], }, diff --git a/src/domains/chat/service.ts b/src/domains/chat/service.ts index 23bfa68..b2c5bce 100644 --- a/src/domains/chat/service.ts +++ b/src/domains/chat/service.ts @@ -85,7 +85,7 @@ export const handleChatTurnEffect = (input: ChatTurnInput) => const readySources = input.sources.filter( (source) => source.status === "ready" && source.knowhereDocumentId, ) - if (readySources.length === 0) { + if (input.sources.length > 0 && readySources.length === 0) { return yield* Effect.fail(noReadySources) } diff --git a/src/domains/chat/types.ts b/src/domains/chat/types.ts index 1d6f88f..e10644f 100644 --- a/src/domains/chat/types.ts +++ b/src/domains/chat/types.ts @@ -5,6 +5,7 @@ export type RetrievalResultView = { readonly content: string readonly chunkType: string readonly score: number | null + readonly chunkId?: string readonly assetUrl?: string readonly pageCitationAssetUrl?: string readonly pageCitationPageNumber?: number diff --git a/src/domains/memory/decay-candidates.test.ts b/src/domains/memory/decay-candidates.test.ts new file mode 100644 index 0000000..7635c1f --- /dev/null +++ b/src/domains/memory/decay-candidates.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest" + +import { selectDecayCandidates } from "./decay-candidates" + +const NOW = new Date("2026-01-08T00:00:00Z") +const A_YEAR_AGO = new Date("2025-01-08T00:00:00Z") + +describe("selectDecayCandidates", () => { + it("flags an old, never-activated item below the threshold", () => { + const candidates = selectDecayCandidates({ + items: [{ id: "item_1", kind: "stance", createdAt: A_YEAR_AGO }], + activationsById: new Map(), + now: NOW, + scoreThreshold: 0.3, + }) + + expect(candidates).toEqual([ + { id: "item_1", kind: "stance", score: expect.any(Number), activationCount: 0 }, + ]) + expect(candidates[0]!.score).toBeLessThan(0.3) + }) + + it("does not flag a freshly created item even with no activations", () => { + const candidates = selectDecayCandidates({ + items: [{ id: "item_1", kind: "stance", createdAt: NOW }], + activationsById: new Map(), + now: NOW, + scoreThreshold: 0.3, + }) + + expect(candidates).toEqual([]) + }) + + it("does not flag an old item that was recently activated", () => { + const candidates = selectDecayCandidates({ + items: [{ id: "item_1", kind: "stance", createdAt: A_YEAR_AGO }], + activationsById: new Map([ + ["item_1", { activationCount: 3, lastActivatedAt: NOW }], + ]), + now: NOW, + scoreThreshold: 0.3, + }) + + expect(candidates).toEqual([]) + }) + + it("only flags items strictly below the given threshold", () => { + const items = [ + { id: "item_1", kind: "stance" as const, createdAt: A_YEAR_AGO }, + { id: "item_2", kind: "stance" as const, createdAt: NOW }, + ] + + const noneFlagged = selectDecayCandidates({ + items, + activationsById: new Map(), + now: NOW, + scoreThreshold: 0, + }) + expect(noneFlagged).toEqual([]) + + const allFlagged = selectDecayCandidates({ + items, + activationsById: new Map(), + now: NOW, + scoreThreshold: 1, + }) + expect(allFlagged.map((c) => c.id).sort()).toEqual(["item_1", "item_2"]) + }) +}) diff --git a/src/domains/memory/decay-candidates.ts b/src/domains/memory/decay-candidates.ts new file mode 100644 index 0000000..159176e --- /dev/null +++ b/src/domains/memory/decay-candidates.ts @@ -0,0 +1,64 @@ +import { computeDecayScore } from "@/domains/retrieval-activation/decay-score" + +/** + * Confirmed decay-candidate threshold: at `BASE_HALF_LIFE_DAYS` (14), a + * never-activated item (ceiling 0.5) crosses this after ~24.3 days of + * silence; one activated once after ~60 days; one activated 3x after ~135 + * days. Chosen by simulating `computeDecayScore`'s actual day-counts across + * activation counts and confirming the resulting grace periods, not picked + * a priori — see the `记忆衰减聚类收尾方案` plan. + */ +export const DEFAULT_DECAY_SCORE_THRESHOLD = 0.15 + +export type DecayableItem = { + readonly id: string + /** Raw `fluid_memory_items.kind` column value — carried through, not validated. */ + readonly kind: string + readonly createdAt: Date +} + +export type ItemActivationStats = { + readonly activationCount: number + readonly lastActivatedAt: Date | null +} + +export type DecayCandidate = { + readonly id: string + readonly kind: string + readonly score: number + readonly activationCount: number +} + +/** + * Pure selection: given active fluid_memory items and their (possibly + * absent) activation ledger rows, return the ones whose decay score is + * below `scoreThreshold`. Does not decide the threshold itself and does not + * write anything — per the plan, a decay score crossing the line only + * produces a *candidate*; moving it to `inactive` is a separate, explicit + * step (see `memoryRepository.deactivateDecayedItemsEffect`). + * + * The anchor for an item with no ledger row (never activated) is its own + * `createdAt` — a real, meaningful signal here (unlike a crystal chunk, + * where "no row" means "no signal at all"), so a never-activated item still + * decays normally from the moment it was created. + */ +export function selectDecayCandidates(input: { + readonly items: readonly DecayableItem[] + readonly activationsById: ReadonlyMap + readonly now: Date + readonly scoreThreshold: number +}): readonly DecayCandidate[] { + return input.items.flatMap((item) => { + const activation = input.activationsById.get(item.id) + const activationCount = activation?.activationCount ?? 0 + const anchorAt = activation?.lastActivatedAt ?? item.createdAt + const score = computeDecayScore({ + activationCount, + anchorAt, + now: input.now, + }) + return score < input.scoreThreshold + ? [{ id: item.id, kind: item.kind, score, activationCount }] + : [] + }) +} diff --git a/src/domains/memory/repository.ts b/src/domains/memory/repository.ts index 3d36662..9b44f87 100644 --- a/src/domains/memory/repository.ts +++ b/src/domains/memory/repository.ts @@ -7,6 +7,7 @@ import type { CapturedObservation } from "./observation-types" import { toDiffOperation, type ResolvedMemoryOperation } from "./resolve-operations" import { buildMemoryItemTokens } from "./search-index" import type { + FluidMemoryDeactivationReason, FluidMemoryKind, FluidMemoryPayload, MemoryDiffOperation, @@ -66,6 +67,24 @@ type MemoryRepository = { readonly deleteExpiredConsumedObservationsEffect: ( olderThan: Date, ) => Effect.Effect + readonly listActiveItemsEffect: ( + workspaceId: string, + ) => Effect.Effect< + readonly Pick[], + never, + DbClient + > + /** + * Move a specific set of active items to `inactive` with reason + * `decayed` (the activation-decay job's candidates, already confirmed by + * the caller — this never decides which items on its own). Mirrors the + * distill `deprecate` write path: drops the item's token rows so it stops + * surfacing as a dedup candidate. + */ + readonly deactivateDecayedItemsEffect: ( + workspaceId: string, + itemIds: readonly string[], + ) => Effect.Effect } type RawRowsResult = readonly Row[] | { readonly rows: readonly Row[] } @@ -257,6 +276,66 @@ const deleteExpiredConsumedObservationsEffect: MemoryRepository["deleteExpiredCo return deleted.length }) +const listActiveItemsEffect: MemoryRepository["listActiveItemsEffect"] = ( + workspaceId, +) => + Effect.gen(function* () { + const db = yield* DbClient + return yield* Effect.promise(() => + db + .select({ + id: fluidMemoryItems.id, + kind: fluidMemoryItems.kind, + createdAt: fluidMemoryItems.createdAt, + }) + .from(fluidMemoryItems) + .where( + and( + eq(fluidMemoryItems.workspaceId, workspaceId), + eq(fluidMemoryItems.status, "active"), + ), + ), + ) + }) + +const deactivateDecayedItemsEffect: MemoryRepository["deactivateDecayedItemsEffect"] = + (workspaceId, itemIds) => + Effect.gen(function* () { + if (itemIds.length === 0) return 0 + + const db = yield* DbClient + return yield* Effect.promise(() => + db.transaction(async (tx) => { + const updated = await tx + .update(fluidMemoryItems) + .set({ + status: "inactive", + deactivationReason: + "decayed" satisfies FluidMemoryDeactivationReason, + updatedAt: sql`now()`, + }) + .where( + and( + eq(fluidMemoryItems.workspaceId, workspaceId), + eq(fluidMemoryItems.status, "active"), + inArray(fluidMemoryItems.id, [...itemIds]), + ), + ) + .returning({ id: fluidMemoryItems.id }) + + if (updated.length > 0) { + await tx.delete(fluidMemoryTokens).where( + inArray( + fluidMemoryTokens.itemId, + updated.map((item) => item.id), + ), + ) + } + return updated.length + }), + ) + }) + export const memoryRepository: MemoryRepository = { findDedupCandidatesEffect, insertObservationsEffect, @@ -264,6 +343,8 @@ export const memoryRepository: MemoryRepository = { listPendingObservationsEffect, applyDistillBatchEffect, deleteExpiredConsumedObservationsEffect, + listActiveItemsEffect, + deactivateDecayedItemsEffect, } async function writeOperations( @@ -353,7 +434,8 @@ async function writeOperations( const [updated] = await tx .update(fluidMemoryItems) .set({ - status: "deprecated", + status: "inactive", + deactivationReason: "contradicted" satisfies FluidMemoryDeactivationReason, updatedAt: sql`now()`, }) .where( diff --git a/src/domains/memory/resolve-operations.test.ts b/src/domains/memory/resolve-operations.test.ts index 403a79a..03484b9 100644 --- a/src/domains/memory/resolve-operations.test.ts +++ b/src/domains/memory/resolve-operations.test.ts @@ -20,7 +20,7 @@ const existingItems = [ }, }, { id: "item-2", kind: "stance", status: "active" }, - { id: "item-3", kind: "stance", status: "deprecated" }, + { id: "item-3", kind: "stance", status: "inactive" }, { id: "item-4", kind: "entity_of_interest", @@ -140,7 +140,7 @@ describe("resolveMemoryOperations", () => { expect(resolved[0]?.op).toBe("skip") }) - it("downgrades merge to skip when the target is already deprecated", () => { + it("downgrades merge to skip when the target is already inactive", () => { const resolved = resolveMemoryOperations({ operations: makeOperations({ stances: [ diff --git a/src/domains/memory/service.ts b/src/domains/memory/service.ts index 41b8b5a..f711f98 100644 --- a/src/domains/memory/service.ts +++ b/src/domains/memory/service.ts @@ -1,12 +1,14 @@ import "server-only" import { databaseRuntime } from "@/domains/workspace/database-runtime" +import { selectDecayCandidates, type DecayCandidate } from "./decay-candidates" import { memoryRepository, type ApplyDistillBatchInput, type InsertObservationsInput, } from "./repository" import type { FluidMemoryKind, MemoryDiffOperation } from "./types" +import { retrievalActivationService } from "@/domains/retrieval-activation/service" import type { FluidMemoryItem, FluidObservation, @@ -34,6 +36,20 @@ type MemoryService = { readonly deleteExpiredConsumedObservations: ( olderThan: Date, ) => Promise + /** + * Active items whose activation-decay score is below `scoreThreshold`. + * Read-only — does not change any item's status. The caller decides the + * threshold and what to do with the result (see + * `deactivateDecayedItems` to actually move candidates to `inactive`). + */ + readonly listDecayCandidates: ( + workspaceId: string, + options: { readonly now: Date; readonly scoreThreshold: number }, + ) => Promise + readonly deactivateDecayedItems: ( + workspaceId: string, + itemIds: readonly string[], + ) => Promise } const findDedupCandidates: MemoryService["findDedupCandidates"] = ( @@ -78,6 +94,46 @@ const deleteExpiredConsumedObservations: MemoryService["deleteExpiredConsumedObs memoryRepository.deleteExpiredConsumedObservationsEffect(olderThan), ) +const listDecayCandidates: MemoryService["listDecayCandidates"] = async ( + workspaceId, + options, +) => { + const items = await databaseRuntime.runPromise( + memoryRepository.listActiveItemsEffect(workspaceId), + ) + if (items.length === 0) return [] + + const activations = await retrievalActivationService.getActivations( + workspaceId, + "fluid_memory", + items.map((item) => item.id), + ) + const activationsById = new Map( + activations.map((activation) => [ + activation.unitRef, + { + activationCount: activation.activationCount, + lastActivatedAt: activation.lastActivatedAt, + }, + ]), + ) + + return selectDecayCandidates({ + items, + activationsById, + now: options.now, + scoreThreshold: options.scoreThreshold, + }) +} + +const deactivateDecayedItems: MemoryService["deactivateDecayedItems"] = ( + workspaceId, + itemIds, +) => + databaseRuntime.runPromise( + memoryRepository.deactivateDecayedItemsEffect(workspaceId, itemIds), + ) + export const memoryService: MemoryService = { findDedupCandidates, insertObservations, @@ -85,4 +141,6 @@ export const memoryService: MemoryService = { listPendingObservations, applyDistillBatch, deleteExpiredConsumedObservations, + listDecayCandidates, + deactivateDecayedItems, } diff --git a/src/domains/memory/types.ts b/src/domains/memory/types.ts index dac7602..b9975cb 100644 --- a/src/domains/memory/types.ts +++ b/src/domains/memory/types.ts @@ -91,6 +91,17 @@ export function parseFluidMemoryPayload( return result.success ? result.data : null } +/** + * Why an item left `active` for `inactive`. Orthogonal to `status`: `status` + * says whether the item is retrievable today, `deactivationReason` says + * which mechanism moved it out. + * - contradicted — distill decided a later turn reverses this item + * - decayed — the activation-decay job flagged it as unused past threshold + */ +export const fluidMemoryDeactivationReasons = ["contradicted", "decayed"] as const +export type FluidMemoryDeactivationReason = + (typeof fluidMemoryDeactivationReasons)[number] + /** One decided operation over the memory set; persisted into memory_diffs. */ export type MemoryDiffOperation = { readonly op: "create" | "skip" | "merge" | "deprecate" diff --git a/src/domains/retrieval-activation/decay-score.test.ts b/src/domains/retrieval-activation/decay-score.test.ts new file mode 100644 index 0000000..db239eb --- /dev/null +++ b/src/domains/retrieval-activation/decay-score.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from "vitest" + +import { BASE_HALF_LIFE_DAYS, computeDecayScore } from "./decay-score" + +const NOW = new Date("2026-01-08T00:00:00Z") + +describe("computeDecayScore", () => { + it("scores a freshly anchored, never-activated unit as neutral 0.5", () => { + // freq = sigmoid(log1p(0)) = sigmoid(0) = 0.5; recency at age 0 = 1. + const score = computeDecayScore({ + activationCount: 0, + anchorAt: NOW, + now: NOW, + }) + expect(score).toBeCloseTo(0.5, 10) + }) + + it("halves the neutral score after one base half-life with no activations", () => { + const anchorAt = new Date( + NOW.getTime() - BASE_HALF_LIFE_DAYS * 24 * 60 * 60 * 1000, + ) + const score = computeDecayScore({ activationCount: 0, anchorAt, now: NOW }) + expect(score).toBeCloseTo(0.25, 10) + }) + + it("decays monotonically with age for a fixed activation count", () => { + const dayAgo = new Date(NOW.getTime() - 24 * 60 * 60 * 1000) + const weekAgo = new Date(NOW.getTime() - 7 * 24 * 60 * 60 * 1000) + + const scoreAtDay = computeDecayScore({ + activationCount: 2, + anchorAt: dayAgo, + now: NOW, + }) + const scoreAtWeek = computeDecayScore({ + activationCount: 2, + anchorAt: weekAgo, + now: NOW, + }) + + expect(scoreAtDay).toBeGreaterThan(scoreAtWeek) + }) + + it("scores a higher activation count above a lower one at the same age", () => { + const weekAgo = new Date(NOW.getTime() - 7 * 24 * 60 * 60 * 1000) + + const lowCount = computeDecayScore({ + activationCount: 1, + anchorAt: weekAgo, + now: NOW, + }) + const highCount = computeDecayScore({ + activationCount: 10, + anchorAt: weekAgo, + now: NOW, + }) + + expect(highCount).toBeGreaterThan(lowCount) + }) + + it("resets toward the frequency ceiling immediately after a fresh activation", () => { + // A unit with a long activation history but an activation just now + // should score close to its frequency ceiling, not its pre-reset decay. + const score = computeDecayScore({ + activationCount: 5, + anchorAt: NOW, + now: NOW, + }) + const frequencyCeiling = 1 / (1 + Math.exp(-Math.log1p(5))) + expect(score).toBeCloseTo(frequencyCeiling, 10) + }) + + it("stays within (0, 1) across a range of counts and ages", () => { + const activationCounts = [0, 1, 3, 10, 50] + const ageDaysList = [0, 1, 7, 30, 365] + + for (const activationCount of activationCounts) { + for (const ageDays of ageDaysList) { + const anchorAt = new Date( + NOW.getTime() - ageDays * 24 * 60 * 60 * 1000, + ) + const score = computeDecayScore({ activationCount, anchorAt, now: NOW }) + expect(score).toBeGreaterThan(0) + expect(score).toBeLessThan(1) + } + } + }) + + it("clamps negative age (anchor in the future) to zero elapsed time", () => { + const future = new Date(NOW.getTime() + 24 * 60 * 60 * 1000) + const score = computeDecayScore({ + activationCount: 0, + anchorAt: future, + now: NOW, + }) + expect(score).toBeCloseTo(0.5, 10) + }) +}) diff --git a/src/domains/retrieval-activation/decay-score.ts b/src/domains/retrieval-activation/decay-score.ts new file mode 100644 index 0000000..0e2ca1b --- /dev/null +++ b/src/domains/retrieval-activation/decay-score.ts @@ -0,0 +1,66 @@ +/** + * Time-decay importance score for a retrievable unit (fluid memory item or + * crystal chunk). Pure function, computed at read time — never persisted — + * matching OpenViking's approach of not storing a score that would need + * migration whenever the formula changes. + * + * Structure copied from OpenViking's `hotness_score` + * (`sigmoid(log1p(activationCount)) × exp(-ln2/halfLife × ageDays)`), see + * `.repos/OpenViking/openviking/retrieve/memory_lifecycle.py`. + * + * `BASE_HALF_LIFE_DAYS` is deliberately 2x OpenViking's own default (7 + * days) — a longer grace period before an unused unit's importance + * meaningfully drops, confirmed against simulated day-counts (see the + * `记忆衰减聚类收尾方案` plan for the numbers this was checked against). + * + * One deviation from OpenViking: the recency half-life grows with + * `activationCount` instead of staying fixed, borrowing MemoryBank's + * (arXiv:2305.10250) intuition that repeated recall makes a memory more + * resistant to forgetting (there, strength `S` is incremented by 1 on every + * recall and used directly as the decay time constant). Here: + * + * effectiveHalfLifeDays = BASE_HALF_LIFE_DAYS * (1 + activationCount) + * + * This does not double-count activationCount with the frequency term: the + * frequency term sets the score's baseline ceiling for a given activation + * count, while the half-life growth slows how fast that ceiling erodes as + * time passes without a new activation. + * + * "Reset then decay" (an activation makes the unit feel fresh again, then + * importance decays again from there) is achieved by the caller advancing + * `anchorAt` to the activation time on every write — this function only + * computes the curve from whatever anchor it is given. + */ + +const MS_PER_DAY = 24 * 60 * 60 * 1000 + +/** 2x OpenViking's DEFAULT_HALF_LIFE_DAYS (7) — see rationale above. */ +export const BASE_HALF_LIFE_DAYS = 14 + +export type DecayScoreInput = { + /** Total times this unit has been cited into an answer. */ + readonly activationCount: number + /** Last activation time, or the unit's creation time if never activated. */ + readonly anchorAt: Date + readonly now: Date +} + +/** Always in (0, 1). */ +export function computeDecayScore(input: DecayScoreInput): number { + const activationCount = Math.max(input.activationCount, 0) + const ageDays = Math.max( + (input.now.getTime() - input.anchorAt.getTime()) / MS_PER_DAY, + 0, + ) + + const frequency = sigmoid(Math.log1p(activationCount)) + const effectiveHalfLifeDays = BASE_HALF_LIFE_DAYS * (1 + activationCount) + const decayRate = Math.LN2 / effectiveHalfLifeDays + const recency = Math.exp(-decayRate * ageDays) + + return frequency * recency +} + +function sigmoid(x: number): number { + return 1 / (1 + Math.exp(-x)) +} diff --git a/src/domains/retrieval-activation/repository.test.ts b/src/domains/retrieval-activation/repository.test.ts new file mode 100644 index 0000000..267c81e --- /dev/null +++ b/src/domains/retrieval-activation/repository.test.ts @@ -0,0 +1,138 @@ +import { afterEach, describe, expect, it, vi } from "vitest" +import { Effect, Layer } from "effect" + +import { retrievalActivationRepository } from "./repository" +import type { Db } from "@/infrastructure/db" + +type InsertValues = { + readonly workspaceId: string + readonly unitType: string + readonly unitRef: string + readonly activationCount: number +} + +afterEach(() => { + vi.restoreAllMocks() +}) + +async function runWithMockDb(insertValues: InsertValues[]) { + const insertBuilder = { + values: vi.fn((values: InsertValues[]) => { + insertValues.push(...values) + return insertBuilder + }), + onConflictDoUpdate: vi.fn(() => insertBuilder), + returning: vi.fn(async () => + insertValues.map((_, index) => ({ id: `activation_${index}` })), + ), + } + const dbMock = { insert: vi.fn(() => insertBuilder) } + const { DbClient } = await vi.importActual( + "@/infrastructure/db", + ) + const dbLayer = Layer.succeed(DbClient, dbMock as unknown as Db) + return { dbLayer, insertBuilder, dbMock } +} + +describe("retrievalActivationRepository.recordActivationsEffect", () => { + it("does nothing and never touches the db for an empty batch", async () => { + const insertValues: InsertValues[] = [] + const { dbLayer, dbMock } = await runWithMockDb(insertValues) + + const written = await Effect.runPromise( + retrievalActivationRepository + .recordActivationsEffect([]) + .pipe(Effect.provide(dbLayer)), + ) + + expect(written).toBe(0) + expect(dbMock.insert).not.toHaveBeenCalled() + }) + + it("collapses duplicate unit refs into a single upsert row", async () => { + const insertValues: InsertValues[] = [] + const { dbLayer, insertBuilder } = await runWithMockDb(insertValues) + + const written = await Effect.runPromise( + retrievalActivationRepository + .recordActivationsEffect([ + { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "doc:chunk_1" }, + { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "doc:chunk_1" }, + { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "doc:chunk_2" }, + ]) + .pipe(Effect.provide(dbLayer)), + ) + + expect(written).toBe(2) + expect(insertBuilder.values).toHaveBeenCalledOnce() + expect(insertValues).toHaveLength(2) + expect(insertValues.map((row) => row.unitRef).sort()).toEqual([ + "doc:chunk_1", + "doc:chunk_2", + ]) + }) + + it("keeps the same unit ref distinct across different unit types and workspaces", async () => { + const insertValues: InsertValues[] = [] + const { dbLayer } = await runWithMockDb(insertValues) + + await Effect.runPromise( + retrievalActivationRepository + .recordActivationsEffect([ + { workspaceId: "ws_1", unitType: "fluid_memory", unitRef: "item_1" }, + { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "item_1" }, + { workspaceId: "ws_2", unitType: "fluid_memory", unitRef: "item_1" }, + ]) + .pipe(Effect.provide(dbLayer)), + ) + + expect(insertValues).toHaveLength(3) + }) +}) + +describe("retrievalActivationRepository.getActivationsEffect", () => { + it("returns [] without querying the db for an empty unit ref list", async () => { + const selectBuilder = { from: vi.fn(), where: vi.fn() } + const dbMock = { select: vi.fn(() => selectBuilder) } + const { DbClient } = await vi.importActual( + "@/infrastructure/db", + ) + const dbLayer = Layer.succeed(DbClient, dbMock as unknown as Db) + + const result = await Effect.runPromise( + retrievalActivationRepository + .getActivationsEffect("ws_1", "fluid_memory", []) + .pipe(Effect.provide(dbLayer)), + ) + + expect(result).toEqual([]) + expect(dbMock.select).not.toHaveBeenCalled() + }) + + it("returns matching activation rows", async () => { + const rows = [ + { + unitRef: "item_1", + activationCount: 3, + lastActivatedAt: new Date("2026-01-01T00:00:00Z"), + }, + ] + const selectBuilder = { + from: vi.fn(() => selectBuilder), + where: vi.fn(async () => rows), + } + const dbMock = { select: vi.fn(() => selectBuilder) } + const { DbClient } = await vi.importActual( + "@/infrastructure/db", + ) + const dbLayer = Layer.succeed(DbClient, dbMock as unknown as Db) + + const result = await Effect.runPromise( + retrievalActivationRepository + .getActivationsEffect("ws_1", "fluid_memory", ["item_1", "item_2"]) + .pipe(Effect.provide(dbLayer)), + ) + + expect(result).toEqual(rows) + }) +}) diff --git a/src/domains/retrieval-activation/repository.ts b/src/domains/retrieval-activation/repository.ts new file mode 100644 index 0000000..53732ac --- /dev/null +++ b/src/domains/retrieval-activation/repository.ts @@ -0,0 +1,119 @@ +import "server-only" + +import { and, eq, inArray, sql } from "drizzle-orm" +import { Effect } from "effect" + +import type { RetrievalUnitType } from "./types" +import { DbClient } from "@/infrastructure/db" +import { retrievalActivations } from "@/infrastructure/db/schema" + +export type RecordActivationInput = { + readonly workspaceId: string + readonly unitType: RetrievalUnitType + readonly unitRef: string +} + +export type ActivationStats = { + readonly unitRef: string + readonly activationCount: number + readonly lastActivatedAt: Date | null +} + +type RetrievalActivationRepository = { + /** + * Upsert one activation (+1, lastActivatedAt = now) per distinct unit. + * Duplicate (workspaceId, unitType, unitRef) entries within `inputs` are + * collapsed to a single +1 — Postgres rejects a multi-row upsert that + * would touch the same conflict target twice in one statement, and + * "cited twice in one answer" should still only count as one activation + * event for this turn. + */ + readonly recordActivationsEffect: ( + inputs: readonly RecordActivationInput[], + ) => Effect.Effect + /** + * Existing ledger rows for a set of unit refs of one type. Units with no + * row (never activated) are simply absent from the result — the caller + * treats that as activationCount 0. + */ + readonly getActivationsEffect: ( + workspaceId: string, + unitType: RetrievalUnitType, + unitRefs: readonly string[], + ) => Effect.Effect +} + +const recordActivationsEffect: RetrievalActivationRepository["recordActivationsEffect"] = + (inputs) => + Effect.gen(function* () { + const deduped = dedupeInputs(inputs) + if (deduped.length === 0) return 0 + + const db = yield* DbClient + const written = yield* Effect.promise(() => + db + .insert(retrievalActivations) + .values( + deduped.map((input) => ({ + workspaceId: input.workspaceId, + unitType: input.unitType, + unitRef: input.unitRef, + activationCount: 1, + lastActivatedAt: sql`now()`, + })), + ) + .onConflictDoUpdate({ + target: [ + retrievalActivations.workspaceId, + retrievalActivations.unitType, + retrievalActivations.unitRef, + ], + set: { + activationCount: sql`${retrievalActivations.activationCount} + 1`, + lastActivatedAt: sql`now()`, + }, + }) + .returning({ id: retrievalActivations.id }), + ) + return written.length + }) + +const getActivationsEffect: RetrievalActivationRepository["getActivationsEffect"] = + (workspaceId, unitType, unitRefs) => + Effect.gen(function* () { + if (unitRefs.length === 0) return [] + + const db = yield* DbClient + const rows = yield* Effect.promise(() => + db + .select({ + unitRef: retrievalActivations.unitRef, + activationCount: retrievalActivations.activationCount, + lastActivatedAt: retrievalActivations.lastActivatedAt, + }) + .from(retrievalActivations) + .where( + and( + eq(retrievalActivations.workspaceId, workspaceId), + eq(retrievalActivations.unitType, unitType), + inArray(retrievalActivations.unitRef, [...unitRefs]), + ), + ), + ) + return rows + }) + +export const retrievalActivationRepository: RetrievalActivationRepository = { + recordActivationsEffect, + getActivationsEffect, +} + +function dedupeInputs( + inputs: readonly RecordActivationInput[], +): RecordActivationInput[] { + const byKey = new Map() + for (const input of inputs) { + byKey.set(`${input.workspaceId}\u0000${input.unitType}\u0000${input.unitRef}`, input) + } + return [...byKey.values()] +} diff --git a/src/domains/retrieval-activation/service.ts b/src/domains/retrieval-activation/service.ts new file mode 100644 index 0000000..990a6eb --- /dev/null +++ b/src/domains/retrieval-activation/service.ts @@ -0,0 +1,45 @@ +import "server-only" + +import { + retrievalActivationRepository, + type ActivationStats, + type RecordActivationInput, +} from "./repository" +import type { RetrievalUnitType } from "./types" +import { databaseRuntime } from "@/domains/workspace/database-runtime" + +type RetrievalActivationService = { + readonly recordActivations: ( + inputs: readonly RecordActivationInput[], + ) => Promise + readonly getActivations: ( + workspaceId: string, + unitType: RetrievalUnitType, + unitRefs: readonly string[], + ) => Promise +} + +const recordActivations: RetrievalActivationService["recordActivations"] = ( + inputs, +) => + databaseRuntime.runPromise( + retrievalActivationRepository.recordActivationsEffect(inputs), + ) + +const getActivations: RetrievalActivationService["getActivations"] = ( + workspaceId, + unitType, + unitRefs, +) => + databaseRuntime.runPromise( + retrievalActivationRepository.getActivationsEffect( + workspaceId, + unitType, + unitRefs, + ), + ) + +export const retrievalActivationService: RetrievalActivationService = { + recordActivations, + getActivations, +} diff --git a/src/domains/retrieval-activation/types.ts b/src/domains/retrieval-activation/types.ts new file mode 100644 index 0000000..29e18db --- /dev/null +++ b/src/domains/retrieval-activation/types.ts @@ -0,0 +1,17 @@ +/** + * A "unit" is anything retrieval can surface and an answer can actually + * cite. Today there are two kinds: + * - fluid_memory — a `fluid_memory_items` row, keyed by its id + * - crystal_chunk — a Knowhere chunk, which has no local row; keyed by + * `${documentId}:${chunkId}` (see `toChunkUnitRef`) + */ +export const retrievalUnitTypes = ["fluid_memory", "crystal_chunk"] as const +export type RetrievalUnitType = (typeof retrievalUnitTypes)[number] + +/** Composite key for a crystal_chunk unit ref. */ +export function toChunkUnitRef(input: { + readonly documentId: string + readonly chunkId: string +}): string { + return `${input.documentId}:${input.chunkId}` +} diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index cb767a0..8963f95 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -358,9 +358,16 @@ export type NewChatMessage = typeof chatMessages.$inferInsert; * one line for pre-filter/dedup context, L1 = short paragraph for later * cognition injection). L2 is the payload itself. * - * Lifecycle: rows start `active`; user revisions deprecate rather than + * Lifecycle: rows start `active`; user revisions deactivate rather than * delete (conservative merge policy), with `version` bumped on merge. * + * `status` is `active` | `inactive`. `inactive` is not itself a + * disambiguated state — `deactivation_reason` records why an item left + * `active` (e.g. `contradicted`, when distill decides a new turn reverses + * this item). This keeps the decay/lifecycle axis (`status`) separate from + * the reason axis, so an activation-decay job can later flip items to + * `inactive` with a different reason without inventing a new status value. + * * `source_message_id` points at the assistant message of the turn the * insight was extracted from; it is set-null on message deletion because * the insight outlives any single turn. @@ -382,6 +389,7 @@ export const fluidMemoryItems = pgTable( ), confidence: doublePrecision("confidence").notNull(), status: text("status").notNull(), + deactivationReason: text("deactivation_reason"), version: integer("version").notNull().default(1), createdAt: timestamp("created_at", { withTimezone: true }) .notNull() @@ -391,7 +399,7 @@ export const fluidMemoryItems = pgTable( .defaultNow(), }, (t) => [ - // Workspace lifecycle scans (active vs deprecated). + // Workspace lifecycle scans (active vs inactive). index("fluid_memory_items_workspace_status_idx").on( t.workspaceId, t.status, @@ -411,8 +419,8 @@ export type NewFluidMemoryItem = typeof fluidMemoryItems.$inferInsert; * * Invariant: token rows exist iff the owning item is `active`. Writers keep * this in sync — create inserts rows, merge replaces them, deprecate deletes - * them — so lookups scan tokens alone (no status join) and never surface a - * deprecated item. + * them — so lookups scan tokens alone (no status join) and never surface an + * inactive item. * * Tokenization mirrors Knowhere map-nav: single CJK characters plus * `[a-z0-9_]+` runs. Scoring is idf-weighted token overlap computed in SQL, @@ -526,3 +534,47 @@ export const fluidObservations = pgTable( export type FluidObservation = typeof fluidObservations.$inferSelect; export type NewFluidObservation = typeof fluidObservations.$inferInsert; + +/** + * Unified activation ledger for retrievable units, driving time-decay + * importance (see src/domains/retrieval-activation/decay-score.ts). + * + * A "unit" is anything that can be surfaced by retrieval and actually cited + * into an answer: today `fluid_memory` (a `fluid_memory_items` row, keyed by + * its id) and `crystal_chunk` (a Knowhere chunk, which has no local row — + * keyed by `${documentId}:${chunkId}`, composed at write time). + * + * Only "really used in an answer" writes here (a citation), not "entered + * the candidate pool" — this avoids overcounting recall as usage. + * + * For `crystal_chunk`, `created_at` is this row's first-write time (the + * first time Notebook observed this chunk being cited), not the chunk's + * true ingestion time in Knowhere — that timestamp is not available to + * Notebook. This is a known, deliberate approximation. + */ +export const retrievalActivations = pgTable( + "retrieval_activations", + { + id: uuid("id").primaryKey().defaultRandom(), + workspaceId: uuid("workspace_id") + .notNull() + .references(() => workspaces.id, { onDelete: "cascade" }), + unitType: text("unit_type").notNull(), + unitRef: text("unit_ref").notNull(), + activationCount: integer("activation_count").notNull().default(0), + lastActivatedAt: timestamp("last_activated_at", { withTimezone: true }), + createdAt: timestamp("created_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (t) => [ + uniqueIndex("retrieval_activations_unit_idx").on( + t.workspaceId, + t.unitType, + t.unitRef, + ), + ], +); + +export type RetrievalActivation = typeof retrievalActivations.$inferSelect; +export type NewRetrievalActivation = typeof retrievalActivations.$inferInsert; From 2d46bc07112d0678d69e0dcd0682401329ecc29e Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Thu, 10 Sep 2026 16:00:33 +0800 Subject: [PATCH 3/4] feat: update environment configuration and integrate new SDK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Updated `.env.local.example` to reflect the new Knowhere API base URL and added Memento service configuration. - Added `@modelcontextprotocol/sdk` dependency to `package.json` and updated `pnpm-lock.yaml` accordingly. - Removed deprecated constants and scripts related to the 观心 case, streamlining the codebase. - Refactored agent harness to utilize the new memory capture functionality, enhancing memory management during chat interactions. - Updated tests to reflect changes in memory handling and ensure proper integration with the new SDK. This update improves the overall functionality and maintainability of the memory and chat systems. --- .env.local.example | 21 +- drizzle/0017_overjoyed_shooting_star.sql | 5 + drizzle/meta/0017_snapshot.json | 932 ++++++++++++++++++ drizzle/meta/_journal.json | 7 + next.config.ts | 1 + package.json | 1 + pnpm-lock.yaml | 460 +++++---- scripts/guanxin-case/constants.ts | 8 - scripts/guanxin-case/ensure-workspace.mts | 21 - scripts/guanxin-case/sample-pilot.py | 91 -- src/agent-harness/runtime.test.ts | 26 +- src/agent-harness/runtime.ts | 27 +- src/app/api/memory/distill/route.ts | 27 - src/app/api/memory/extract/route.ts | 27 - src/domains/chat/commit-turn.test.ts | 25 +- src/domains/chat/commit-turn.ts | 68 +- src/domains/chat/memory-tools.test.ts | 104 -- src/domains/chat/memory-tools.ts | 66 -- src/domains/chat/prompt.ts | 4 +- src/domains/chat/route-service.test.ts | 17 +- src/domains/chat/service.test.ts | 32 +- src/domains/chat/service.ts | 2 +- src/domains/memory/decay-candidates.test.ts | 69 -- src/domains/memory/decay-candidates.ts | 64 -- src/domains/memory/distill-config.ts | 19 - src/domains/memory/distill-model.ts | 83 -- src/domains/memory/distill-prompts.test.ts | 110 --- src/domains/memory/distill-prompts.ts | 311 ------ src/domains/memory/distill-trigger.test.ts | 84 -- src/domains/memory/distill-trigger.ts | 77 -- src/domains/memory/distill-types.test.ts | 209 ---- src/domains/memory/distill-types.ts | 190 ---- src/domains/memory/distill-workflow.test.ts | 48 - src/domains/memory/distill-workflow.ts | 231 ----- src/domains/memory/extract-trigger.ts | 43 - src/domains/memory/extract-workflow.test.ts | 40 - src/domains/memory/extract-workflow.ts | 128 --- src/domains/memory/extraction-model.ts | 50 - src/domains/memory/observation-types.test.ts | 78 -- src/domains/memory/observation-types.ts | 50 - src/domains/memory/prompts.test.ts | 54 - src/domains/memory/prompts.ts | 102 -- src/domains/memory/repository.ts | 500 ---------- src/domains/memory/resolve-operations.test.ts | 378 ------- src/domains/memory/resolve-operations.ts | 314 ------ src/domains/memory/search-index.test.ts | 94 -- src/domains/memory/search-index.ts | 84 -- src/domains/memory/service.ts | 146 --- src/domains/memory/types.ts | 112 --- .../retrieval-activation/decay-score.test.ts | 98 -- .../retrieval-activation/decay-score.ts | 66 -- .../retrieval-activation/repository.test.ts | 138 --- .../retrieval-activation/repository.ts | 119 --- src/domains/retrieval-activation/service.ts | 45 - src/domains/retrieval-activation/types.ts | 17 - src/infrastructure/db/schema.ts | 240 ----- src/integrations/memento/client.ts | 75 ++ src/integrations/memento/config.ts | 18 + src/integrations/memento/memory-tools.ts | 105 ++ 59 files changed, 1499 insertions(+), 5062 deletions(-) create mode 100644 drizzle/0017_overjoyed_shooting_star.sql create mode 100644 drizzle/meta/0017_snapshot.json delete mode 100644 scripts/guanxin-case/constants.ts delete mode 100644 scripts/guanxin-case/ensure-workspace.mts delete mode 100644 scripts/guanxin-case/sample-pilot.py delete mode 100644 src/app/api/memory/distill/route.ts delete mode 100644 src/app/api/memory/extract/route.ts delete mode 100644 src/domains/chat/memory-tools.test.ts delete mode 100644 src/domains/chat/memory-tools.ts delete mode 100644 src/domains/memory/decay-candidates.test.ts delete mode 100644 src/domains/memory/decay-candidates.ts delete mode 100644 src/domains/memory/distill-config.ts delete mode 100644 src/domains/memory/distill-model.ts delete mode 100644 src/domains/memory/distill-prompts.test.ts delete mode 100644 src/domains/memory/distill-prompts.ts delete mode 100644 src/domains/memory/distill-trigger.test.ts delete mode 100644 src/domains/memory/distill-trigger.ts delete mode 100644 src/domains/memory/distill-types.test.ts delete mode 100644 src/domains/memory/distill-types.ts delete mode 100644 src/domains/memory/distill-workflow.test.ts delete mode 100644 src/domains/memory/distill-workflow.ts delete mode 100644 src/domains/memory/extract-trigger.ts delete mode 100644 src/domains/memory/extract-workflow.test.ts delete mode 100644 src/domains/memory/extract-workflow.ts delete mode 100644 src/domains/memory/extraction-model.ts delete mode 100644 src/domains/memory/observation-types.test.ts delete mode 100644 src/domains/memory/observation-types.ts delete mode 100644 src/domains/memory/prompts.test.ts delete mode 100644 src/domains/memory/prompts.ts delete mode 100644 src/domains/memory/repository.ts delete mode 100644 src/domains/memory/resolve-operations.test.ts delete mode 100644 src/domains/memory/resolve-operations.ts delete mode 100644 src/domains/memory/search-index.test.ts delete mode 100644 src/domains/memory/search-index.ts delete mode 100644 src/domains/memory/service.ts delete mode 100644 src/domains/memory/types.ts delete mode 100644 src/domains/retrieval-activation/decay-score.test.ts delete mode 100644 src/domains/retrieval-activation/decay-score.ts delete mode 100644 src/domains/retrieval-activation/repository.test.ts delete mode 100644 src/domains/retrieval-activation/repository.ts delete mode 100644 src/domains/retrieval-activation/service.ts delete mode 100644 src/domains/retrieval-activation/types.ts create mode 100644 src/integrations/memento/client.ts create mode 100644 src/integrations/memento/config.ts create mode 100644 src/integrations/memento/memory-tools.ts diff --git a/.env.local.example b/.env.local.example index 1307872..097692f 100644 --- a/.env.local.example +++ b/.env.local.example @@ -1,11 +1,17 @@ -# Optional API base URL. Defaults to production when unset. -# Use staging when validating staging keys. -# KNOWHERE_BASE_URL=https://api-staging.knowhereto.ai +# Knowhere API origin. Unset also means production. +# Local recommendation: production. Staging is only for staging keys. +KNOWHERE_BASE_URL=https://api.knowhereto.ai -# Optional development override. When set, Notebook skips Dashboard session +# Local development override. When set, Notebook skips Dashboard session # auth and Dashboard-issued JWT creation, then calls Knowhere directly with # this key. Leave unset for production and Dashboard-authenticated staging. -# KNOWHERE_API_KEY=sk_your_development_key_here +# KNOWHERE_API_KEY= + +# --- Local demo case only (观心). Not Notebook core. --- +# Put the real key in gitignored `.env.local`, copied from +# `观心2.0-RAG-v1.1-demo/_retrieval/.env`. +# Do not copy that file's KNOWHERE_BASE_URL — it is a retrieval endpoint +# path, not the SDK origin above. # --- Chat provider (server-side only) --- # Vercel AI Gateway key; AI SDK picks it up automatically @@ -61,3 +67,8 @@ DATABASE_URL=postgres://user:password@host/db # pg — postgres-js (use for local dev against a plain Postgres, # or for AWS Aurora Postgres if/when we migrate off Neon) DATABASE_DRIVER=pg + +# --- Memento (fluid memory + retrieval activation) --- +# Standalone REST + MCP service. Notebook no longer owns these tables. +MEMENTO_BASE_URL=http://localhost:8787 +MEMENTO_SERVICE_KEY= diff --git a/drizzle/0017_overjoyed_shooting_star.sql b/drizzle/0017_overjoyed_shooting_star.sql new file mode 100644 index 0000000..6d97d3a --- /dev/null +++ b/drizzle/0017_overjoyed_shooting_star.sql @@ -0,0 +1,5 @@ +DROP TABLE "fluid_memory_items" CASCADE;--> statement-breakpoint +DROP TABLE "fluid_memory_tokens" CASCADE;--> statement-breakpoint +DROP TABLE "fluid_observations" CASCADE;--> statement-breakpoint +DROP TABLE "memory_diffs" CASCADE;--> statement-breakpoint +DROP TABLE "retrieval_activations" CASCADE; \ No newline at end of file diff --git a/drizzle/meta/0017_snapshot.json b/drizzle/meta/0017_snapshot.json new file mode 100644 index 0000000..52d8823 --- /dev/null +++ b/drizzle/meta/0017_snapshot.json @@ -0,0 +1,932 @@ +{ + "id": "d4de5ce8-e16b-4dc2-a9ad-2b76bbca8150", + "prevId": "55d73dc7-9826-4cb5-9bb6-3506b8fa338a", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parsed_document_sync_leases": { + "name": "parsed_document_sync_leases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_token": { + "name": "lease_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_reason": { + "name": "release_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parsed_document_sync_leases_token_idx": { + "name": "parsed_document_sync_leases_token_idx", + "columns": [ + { + "expression": "lease_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_active_idx": { + "name": "parsed_document_sync_leases_active_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_workspace_active_idx": { + "name": "parsed_document_sync_leases_workspace_active_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_document_active_idx": { + "name": "parsed_document_sync_leases_document_active_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parsed_document_sync_leases_workspace_id_workspaces_id_fk": { + "name": "parsed_document_sync_leases_workspace_id_workspaces_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parsed_document_sync_leases_source_id_sources_id_fk": { + "name": "parsed_document_sync_leases_source_id_sources_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index a8f7671..4ee28af 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -120,6 +120,13 @@ "when": 1788952172906, "tag": "0016_wealthy_matthew_murdock", "breakpoints": true + }, + { + "idx": 17, + "version": "7", + "when": 1789017481605, + "tag": "0017_overjoyed_shooting_star", + "breakpoints": true } ] } \ No newline at end of file diff --git a/next.config.ts b/next.config.ts index b73f520..cd1fac0 100644 --- a/next.config.ts +++ b/next.config.ts @@ -10,6 +10,7 @@ const nextConfig: NextConfig = { "@ontos-ai/knowhere-sdk", "@napi-rs/canvas", "piscina", + "@modelcontextprotocol/sdk", ], allowedDevOrigins: [ "127.0.0.1", diff --git a/package.json b/package.json index f377889..4ab2532 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "@ai-sdk/react": "^3.0.177", "@antv/chart-visualization-skills": "0.1.3", "@effect/platform": "^0.96.1", + "@modelcontextprotocol/sdk": "^1.30.0", "@napi-rs/canvas": "^1.0.2", "@neondatabase/serverless": "^1.1.0", "@ontos-ai/knowhere-sdk": "^2.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8847510..d4db84c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: '@effect/platform': specifier: ^0.96.1 version: 0.96.1(effect@3.21.2) + '@modelcontextprotocol/sdk': + specifier: ^1.30.0 + version: 1.30.0(supports-color@7.2.0)(zod@4.4.3) '@napi-rs/canvas': specifier: ^1.0.2 version: 1.0.2 @@ -25,7 +28,7 @@ importers: version: 1.1.0 '@ontos-ai/knowhere-sdk': specifier: ^2.2.0 - version: 2.2.0 + version: 2.2.0(debug@4.4.3(supports-color@7.2.0))(supports-color@7.2.0) '@radix-ui/react-alert-dialog': specifier: ^1.1.15 version: 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -97,7 +100,7 @@ importers: version: 1.12.0 next: specifier: 16.2.4 - version: 16.2.4(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + version: 16.2.4(@babel/core@7.29.0(supports-color@7.2.0))(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) next-themes: specifier: ^0.4.6 version: 0.4.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -121,13 +124,13 @@ importers: version: 19.2.4(react@19.2.4) react-markdown: specifier: ^10.1.0 - version: 10.1.0(@types/react@19.2.14)(react@19.2.4) + version: 10.1.0(@types/react@19.2.14)(react@19.2.4)(supports-color@7.2.0) react-pdf: specifier: ^10.4.1 version: 10.4.1(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) remark-gfm: specifier: ^4.0.1 - version: 4.0.1 + version: 4.0.1(supports-color@7.2.0) server-only: specifier: ^0.0.1 version: 0.0.1 @@ -182,16 +185,16 @@ importers: version: 0.31.10 eslint: specifier: ^9 - version: 9.39.4(jiti@2.7.0) + version: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) eslint-config-next: specifier: 16.2.4 - version: 16.2.4(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) + version: 16.2.4(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) jsdom: specifier: ^29.1.1 version: 29.1.1(@noble/hashes@1.8.0) shadcn: specifier: ^4.7.0 - version: 4.7.0(@types/node@20.19.39)(typescript@6.0.3) + version: 4.7.0(@types/node@20.19.39)(supports-color@7.2.0)(typescript@6.0.3) tailwindcss: specifier: ^4 version: 4.2.4 @@ -455,11 +458,11 @@ packages: '@esbuild-kit/core-utils@3.3.2': resolution: {integrity: sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild-kit/esm-loader@2.6.5': resolution: {integrity: sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild/aix-ppc64@0.25.12': resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} @@ -1197,8 +1200,8 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} - '@modelcontextprotocol/sdk@1.29.0': - resolution: {integrity: sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==} + '@modelcontextprotocol/sdk@1.30.0': + resolution: {integrity: sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==} engines: {node: '>=18'} peerDependencies: '@cfworker/json-schema': ^4.1.1 @@ -2594,6 +2597,7 @@ packages: '@xmldom/xmldom@0.8.13': resolution: {integrity: sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==} engines: {node: '>=10.0.0'} + deprecated: this version has critical issues, please update to the latest version accepts@2.0.0: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} @@ -2948,6 +2952,7 @@ packages: crypto-js@4.2.0: resolution: {integrity: sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==} + deprecated: Active development of CryptoJS has been discontinued. This library is no longer maintained. css-tree@3.2.1: resolution: {integrity: sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==} @@ -3394,6 +3399,7 @@ packages: eslint@9.39.4: resolution: {integrity: sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true peerDependencies: jiti: '*' @@ -5733,20 +5739,20 @@ snapshots: '@babel/compat-data@7.29.3': {} - '@babel/core@7.29.0': + '@babel/core@7.29.0(supports-color@7.2.0)': dependencies: '@babel/code-frame': 7.29.0 '@babel/generator': 7.29.1 '@babel/helper-compilation-targets': 7.28.6 - '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0) + '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helpers': 7.29.2 '@babel/parser': 7.29.3 '@babel/template': 7.28.6 - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) '@babel/types': 7.29.0 '@jridgewell/remapping': 2.3.5 convert-source-map: 2.0.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -5773,41 +5779,41 @@ snapshots: lru-cache: 5.1.1 semver: 6.3.1 - '@babel/helper-create-class-features-plugin@7.29.3(@babel/core@7.29.0)': + '@babel/helper-create-class-features-plugin@7.29.3(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/helper-annotate-as-pure': 7.27.3 - '@babel/helper-member-expression-to-functions': 7.28.5 + '@babel/helper-member-expression-to-functions': 7.28.5(supports-color@7.2.0) '@babel/helper-optimise-call-expression': 7.27.1 - '@babel/helper-replace-supers': 7.28.6(@babel/core@7.29.0) - '@babel/helper-skip-transparent-expression-wrappers': 7.27.1 - '@babel/traverse': 7.29.0 + '@babel/helper-replace-supers': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/helper-skip-transparent-expression-wrappers': 7.27.1(supports-color@7.2.0) + '@babel/traverse': 7.29.0(supports-color@7.2.0) semver: 6.3.1 transitivePeerDependencies: - supports-color '@babel/helper-globals@7.28.0': {} - '@babel/helper-member-expression-to-functions@7.28.5': + '@babel/helper-member-expression-to-functions@7.28.5(supports-color@7.2.0)': dependencies: - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) '@babel/types': 7.29.0 transitivePeerDependencies: - supports-color - '@babel/helper-module-imports@7.28.6': + '@babel/helper-module-imports@7.28.6(supports-color@7.2.0)': dependencies: - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) '@babel/types': 7.29.0 transitivePeerDependencies: - supports-color - '@babel/helper-module-transforms@7.28.6(@babel/core@7.29.0)': + '@babel/helper-module-transforms@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 - '@babel/helper-module-imports': 7.28.6 + '@babel/core': 7.29.0(supports-color@7.2.0) + '@babel/helper-module-imports': 7.28.6(supports-color@7.2.0) '@babel/helper-validator-identifier': 7.28.5 - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -5817,18 +5823,18 @@ snapshots: '@babel/helper-plugin-utils@7.28.6': {} - '@babel/helper-replace-supers@7.28.6(@babel/core@7.29.0)': + '@babel/helper-replace-supers@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 - '@babel/helper-member-expression-to-functions': 7.28.5 + '@babel/core': 7.29.0(supports-color@7.2.0) + '@babel/helper-member-expression-to-functions': 7.28.5(supports-color@7.2.0) '@babel/helper-optimise-call-expression': 7.27.1 - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) transitivePeerDependencies: - supports-color - '@babel/helper-skip-transparent-expression-wrappers@7.27.1': + '@babel/helper-skip-transparent-expression-wrappers@7.27.1(supports-color@7.2.0)': dependencies: - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) '@babel/types': 7.29.0 transitivePeerDependencies: - supports-color @@ -5848,43 +5854,43 @@ snapshots: dependencies: '@babel/types': 7.29.0 - '@babel/plugin-syntax-jsx@7.28.6(@babel/core@7.29.0)': + '@babel/plugin-syntax-jsx@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))': dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.28.6 - '@babel/plugin-syntax-typescript@7.28.6(@babel/core@7.29.0)': + '@babel/plugin-syntax-typescript@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))': dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.28.6 - '@babel/plugin-transform-modules-commonjs@7.28.6(@babel/core@7.29.0)': + '@babel/plugin-transform-modules-commonjs@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 - '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0) + '@babel/core': 7.29.0(supports-color@7.2.0) + '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.28.6 transitivePeerDependencies: - supports-color - '@babel/plugin-transform-typescript@7.28.6(@babel/core@7.29.0)': + '@babel/plugin-transform-typescript@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/helper-annotate-as-pure': 7.27.3 - '@babel/helper-create-class-features-plugin': 7.29.3(@babel/core@7.29.0) + '@babel/helper-create-class-features-plugin': 7.29.3(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.28.6 - '@babel/helper-skip-transparent-expression-wrappers': 7.27.1 - '@babel/plugin-syntax-typescript': 7.28.6(@babel/core@7.29.0) + '@babel/helper-skip-transparent-expression-wrappers': 7.27.1(supports-color@7.2.0) + '@babel/plugin-syntax-typescript': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0)) transitivePeerDependencies: - supports-color - '@babel/preset-typescript@7.28.5(@babel/core@7.29.0)': + '@babel/preset-typescript@7.28.5(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.28.6 '@babel/helper-validator-option': 7.27.1 - '@babel/plugin-syntax-jsx': 7.28.6(@babel/core@7.29.0) - '@babel/plugin-transform-modules-commonjs': 7.28.6(@babel/core@7.29.0) - '@babel/plugin-transform-typescript': 7.28.6(@babel/core@7.29.0) + '@babel/plugin-syntax-jsx': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0)) + '@babel/plugin-transform-modules-commonjs': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/plugin-transform-typescript': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -5896,7 +5902,7 @@ snapshots: '@babel/parser': 7.29.3 '@babel/types': 7.29.0 - '@babel/traverse@7.29.0': + '@babel/traverse@7.29.0(supports-color@7.2.0)': dependencies: '@babel/code-frame': 7.29.0 '@babel/generator': 7.29.1 @@ -5904,7 +5910,7 @@ snapshots: '@babel/parser': 7.29.3 '@babel/template': 7.28.6 '@babel/types': 7.29.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -6217,17 +6223,17 @@ snapshots: '@esbuild/win32-x64@0.27.7': optional: true - '@eslint-community/eslint-utils@4.9.1(eslint@9.39.4(jiti@2.7.0))': + '@eslint-community/eslint-utils@4.9.1(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))': dependencies: - eslint: 9.39.4(jiti@2.7.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) eslint-visitor-keys: 3.4.3 '@eslint-community/regexpp@4.12.2': {} - '@eslint/config-array@0.21.2': + '@eslint/config-array@0.21.2(supports-color@7.2.0)': dependencies: '@eslint/object-schema': 2.1.7 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) minimatch: 3.1.5 transitivePeerDependencies: - supports-color @@ -6240,10 +6246,10 @@ snapshots: dependencies: '@types/json-schema': 7.0.15 - '@eslint/eslintrc@3.3.5': + '@eslint/eslintrc@3.3.5(supports-color@7.2.0)': dependencies: ajv: 6.15.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) espree: 10.4.0 globals: 14.0.0 ignore: 5.3.2 @@ -6447,7 +6453,7 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 - '@modelcontextprotocol/sdk@1.29.0(zod@3.25.76)': + '@modelcontextprotocol/sdk@1.30.0(supports-color@7.2.0)(zod@3.25.76)': dependencies: '@hono/node-server': 1.19.14(hono@4.12.17) ajv: 8.20.0 @@ -6457,8 +6463,8 @@ snapshots: cross-spawn: 7.0.6 eventsource: 3.0.7 eventsource-parser: 3.0.8 - express: 5.2.1 - express-rate-limit: 8.5.0(express@5.2.1) + express: 5.2.1(supports-color@7.2.0) + express-rate-limit: 8.5.0(express@5.2.1(supports-color@7.2.0)) hono: 4.12.17 jose: 6.2.3 json-schema-typed: 8.0.2 @@ -6469,6 +6475,28 @@ snapshots: transitivePeerDependencies: - supports-color + '@modelcontextprotocol/sdk@1.30.0(supports-color@7.2.0)(zod@4.4.3)': + dependencies: + '@hono/node-server': 1.19.14(hono@4.12.17) + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) + content-type: 1.0.5 + cors: 2.8.6 + cross-spawn: 7.0.6 + eventsource: 3.0.7 + eventsource-parser: 3.0.8 + express: 5.2.1(supports-color@7.2.0) + express-rate-limit: 8.5.0(express@5.2.1(supports-color@7.2.0)) + hono: 4.12.17 + jose: 6.2.3 + json-schema-typed: 8.0.2 + pkce-challenge: 5.0.1 + raw-body: 3.0.2 + zod: 4.4.3 + zod-to-json-schema: 3.25.2(zod@4.4.3) + transitivePeerDependencies: + - supports-color + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': optional: true @@ -6731,9 +6759,9 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} - '@ontos-ai/knowhere-sdk@2.2.0': + '@ontos-ai/knowhere-sdk@2.2.0(debug@4.4.3(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - axios: 1.18.1 + axios: 1.18.1(debug@4.4.3(supports-color@7.2.0))(supports-color@7.2.0) jszip: 3.10.1 transitivePeerDependencies: - debug @@ -7400,15 +7428,15 @@ snapshots: '@types/validate-npm-package-name@4.0.2': {} - '@typescript-eslint/eslint-plugin@8.59.2(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/eslint-plugin@8.59.2(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3)': dependencies: '@eslint-community/regexpp': 4.12.2 - '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) '@typescript-eslint/scope-manager': 8.59.2 - '@typescript-eslint/type-utils': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/type-utils': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) '@typescript-eslint/visitor-keys': 8.59.2 - eslint: 9.39.4(jiti@2.7.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) ignore: 7.0.5 natural-compare: 1.4.0 ts-api-utils: 2.5.0(typescript@6.0.3) @@ -7416,23 +7444,23 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3)': dependencies: '@typescript-eslint/scope-manager': 8.59.2 '@typescript-eslint/types': 8.59.2 - '@typescript-eslint/typescript-estree': 8.59.2(typescript@6.0.3) + '@typescript-eslint/typescript-estree': 8.59.2(supports-color@7.2.0)(typescript@6.0.3) '@typescript-eslint/visitor-keys': 8.59.2 - debug: 4.4.3 - eslint: 9.39.4(jiti@2.7.0) + debug: 4.4.3(supports-color@7.2.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color - '@typescript-eslint/project-service@8.59.2(typescript@6.0.3)': + '@typescript-eslint/project-service@8.59.2(supports-color@7.2.0)(typescript@6.0.3)': dependencies: '@typescript-eslint/tsconfig-utils': 8.59.2(typescript@6.0.3) '@typescript-eslint/types': 8.59.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -7446,13 +7474,13 @@ snapshots: dependencies: typescript: 6.0.3 - '@typescript-eslint/type-utils@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/type-utils@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3)': dependencies: '@typescript-eslint/types': 8.59.2 - '@typescript-eslint/typescript-estree': 8.59.2(typescript@6.0.3) - '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - debug: 4.4.3 - eslint: 9.39.4(jiti@2.7.0) + '@typescript-eslint/typescript-estree': 8.59.2(supports-color@7.2.0)(typescript@6.0.3) + '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + debug: 4.4.3(supports-color@7.2.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) ts-api-utils: 2.5.0(typescript@6.0.3) typescript: 6.0.3 transitivePeerDependencies: @@ -7460,13 +7488,13 @@ snapshots: '@typescript-eslint/types@8.59.2': {} - '@typescript-eslint/typescript-estree@8.59.2(typescript@6.0.3)': + '@typescript-eslint/typescript-estree@8.59.2(supports-color@7.2.0)(typescript@6.0.3)': dependencies: - '@typescript-eslint/project-service': 8.59.2(typescript@6.0.3) + '@typescript-eslint/project-service': 8.59.2(supports-color@7.2.0)(typescript@6.0.3) '@typescript-eslint/tsconfig-utils': 8.59.2(typescript@6.0.3) '@typescript-eslint/types': 8.59.2 '@typescript-eslint/visitor-keys': 8.59.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) minimatch: 10.2.5 semver: 7.7.4 tinyglobby: 0.2.16 @@ -7475,13 +7503,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/utils@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3)': dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.7.0)) + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)) '@typescript-eslint/scope-manager': 8.59.2 '@typescript-eslint/types': 8.59.2 - '@typescript-eslint/typescript-estree': 8.59.2(typescript@6.0.3) - eslint: 9.39.4(jiti@2.7.0) + '@typescript-eslint/typescript-estree': 8.59.2(supports-color@7.2.0)(typescript@6.0.3) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -7635,9 +7663,9 @@ snapshots: acorn@8.16.0: {} - agent-base@6.0.2: + agent-base@6.0.2(supports-color@7.2.0): dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -7784,11 +7812,11 @@ snapshots: axe-core@4.11.4: {} - axios@1.18.1: + axios@1.18.1(debug@4.4.3(supports-color@7.2.0))(supports-color@7.2.0): dependencies: - follow-redirects: 1.16.0 + follow-redirects: 1.16.0(debug@4.4.3(supports-color@7.2.0)) form-data: 4.0.6 - https-proxy-agent: 5.0.1 + https-proxy-agent: 5.0.1(supports-color@7.2.0) proxy-from-env: 2.1.0 transitivePeerDependencies: - debug @@ -7816,11 +7844,11 @@ snapshots: bluebird@3.4.7: {} - body-parser@2.2.2: + body-parser@2.2.2(supports-color@7.2.0): dependencies: bytes: 3.1.2 content-type: 1.0.5 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) http-errors: 2.0.1 iconv-lite: 0.7.2 on-finished: 2.4.1 @@ -8021,13 +8049,17 @@ snapshots: es-errors: 1.3.0 is-data-view: 1.0.2 - debug@3.2.7: + debug@3.2.7(supports-color@7.2.0): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 7.2.0 - debug@4.4.3: + debug@4.4.3(supports-color@7.2.0): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 7.2.0 decimal.js@10.6.0: {} @@ -8350,18 +8382,18 @@ snapshots: escape-string-regexp@5.0.0: {} - eslint-config-next@16.2.4(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3): + eslint-config-next@16.2.4(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3): dependencies: '@next/eslint-plugin-next': 16.2.4 - eslint: 9.39.4(jiti@2.7.0) - eslint-import-resolver-node: 0.3.10 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)) - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)) - eslint-plugin-jsx-a11y: 6.10.2(eslint@9.39.4(jiti@2.7.0)) - eslint-plugin-react: 7.37.5(eslint@9.39.4(jiti@2.7.0)) - eslint-plugin-react-hooks: 7.1.1(eslint@9.39.4(jiti@2.7.0)) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) + eslint-import-resolver-node: 0.3.10(supports-color@7.2.0) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) + eslint-plugin-jsx-a11y: 6.10.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)) + eslint-plugin-react: 7.37.5(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)) + eslint-plugin-react-hooks: 7.1.1(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) globals: 16.4.0 - typescript-eslint: 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) + typescript-eslint: 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) optionalDependencies: typescript: 6.0.3 transitivePeerDependencies: @@ -8370,52 +8402,52 @@ snapshots: - eslint-plugin-import-x - supports-color - eslint-import-resolver-node@0.3.10: + eslint-import-resolver-node@0.3.10(supports-color@7.2.0): dependencies: - debug: 3.2.7 + debug: 3.2.7(supports-color@7.2.0) is-core-module: 2.16.2 resolve: 2.0.0-next.6 transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)): + eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0): dependencies: '@nolyfill/is-core-module': 1.0.39 - debug: 4.4.3 - eslint: 9.39.4(jiti@2.7.0) + debug: 4.4.3(supports-color@7.2.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) get-tsconfig: 4.14.0 is-bun-module: 2.0.0 stable-hash: 0.0.5 tinyglobby: 0.2.16 unrs-resolver: 1.11.1 optionalDependencies: - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.1(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-node@0.3.10)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)): + eslint-module-utils@2.12.1(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint-import-resolver-node@0.3.10(supports-color@7.2.0))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0): dependencies: - debug: 3.2.7 + debug: 3.2.7(supports-color@7.2.0) optionalDependencies: - '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - eslint: 9.39.4(jiti@2.7.0) - eslint-import-resolver-node: 0.3.10 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)) + '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) + eslint-import-resolver-node: 0.3.10(supports-color@7.2.0) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) transitivePeerDependencies: - supports-color - eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)): + eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0): dependencies: '@rtsao/scc': 1.1.0 array-includes: 3.1.9 array.prototype.findlastindex: 1.2.6 array.prototype.flat: 1.3.3 array.prototype.flatmap: 1.3.3 - debug: 3.2.7 + debug: 3.2.7(supports-color@7.2.0) doctrine: 2.1.0 - eslint: 9.39.4(jiti@2.7.0) - eslint-import-resolver-node: 0.3.10 - eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-node@0.3.10)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) + eslint-import-resolver-node: 0.3.10(supports-color@7.2.0) + eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint-import-resolver-node@0.3.10(supports-color@7.2.0))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) hasown: 2.0.3 is-core-module: 2.16.2 is-glob: 4.0.3 @@ -8427,13 +8459,13 @@ snapshots: string.prototype.trimend: 1.0.9 tsconfig-paths: 3.15.0 optionalDependencies: - '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) transitivePeerDependencies: - eslint-import-resolver-typescript - eslint-import-resolver-webpack - supports-color - eslint-plugin-jsx-a11y@6.10.2(eslint@9.39.4(jiti@2.7.0)): + eslint-plugin-jsx-a11y@6.10.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)): dependencies: aria-query: 5.3.2 array-includes: 3.1.9 @@ -8443,7 +8475,7 @@ snapshots: axobject-query: 4.1.0 damerau-levenshtein: 1.0.8 emoji-regex: 9.2.2 - eslint: 9.39.4(jiti@2.7.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) hasown: 2.0.3 jsx-ast-utils: 3.3.5 language-tags: 1.0.9 @@ -8452,18 +8484,18 @@ snapshots: safe-regex-test: 1.1.0 string.prototype.includes: 2.0.1 - eslint-plugin-react-hooks@7.1.1(eslint@9.39.4(jiti@2.7.0)): + eslint-plugin-react-hooks@7.1.1(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0): dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/parser': 7.29.3 - eslint: 9.39.4(jiti@2.7.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) hermes-parser: 0.25.1 zod: 4.4.3 zod-validation-error: 4.0.2(zod@4.4.3) transitivePeerDependencies: - supports-color - eslint-plugin-react@7.37.5(eslint@9.39.4(jiti@2.7.0)): + eslint-plugin-react@7.37.5(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)): dependencies: array-includes: 3.1.9 array.prototype.findlast: 1.2.5 @@ -8471,7 +8503,7 @@ snapshots: array.prototype.tosorted: 1.1.4 doctrine: 2.1.0 es-iterator-helpers: 1.3.2 - eslint: 9.39.4(jiti@2.7.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) estraverse: 5.3.0 hasown: 2.0.3 jsx-ast-utils: 3.3.5 @@ -8496,14 +8528,14 @@ snapshots: eslint-visitor-keys@5.0.1: {} - eslint@9.39.4(jiti@2.7.0): + eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0): dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.7.0)) + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)) '@eslint-community/regexpp': 4.12.2 - '@eslint/config-array': 0.21.2 + '@eslint/config-array': 0.21.2(supports-color@7.2.0) '@eslint/config-helpers': 0.4.2 '@eslint/core': 0.17.0 - '@eslint/eslintrc': 3.3.5 + '@eslint/eslintrc': 3.3.5(supports-color@7.2.0) '@eslint/js': 9.39.4 '@eslint/plugin-kit': 0.4.1 '@humanfs/node': 0.16.8 @@ -8513,7 +8545,7 @@ snapshots: ajv: 6.15.0 chalk: 4.1.2 cross-spawn: 7.0.6 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) escape-string-regexp: 4.0.0 eslint-scope: 8.4.0 eslint-visitor-keys: 4.2.1 @@ -8600,25 +8632,25 @@ snapshots: expect-type@1.3.0: {} - express-rate-limit@8.5.0(express@5.2.1): + express-rate-limit@8.5.0(express@5.2.1(supports-color@7.2.0)): dependencies: - express: 5.2.1 + express: 5.2.1(supports-color@7.2.0) ip-address: 10.1.0 - express@5.2.1: + express@5.2.1(supports-color@7.2.0): dependencies: accepts: 2.0.0 - body-parser: 2.2.2 + body-parser: 2.2.2(supports-color@7.2.0) content-disposition: 1.1.0 content-type: 1.0.5 cookie: 0.7.2 cookie-signature: 1.2.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) depd: 2.0.0 encodeurl: 2.0.0 escape-html: 1.0.3 etag: 1.8.1 - finalhandler: 2.1.1 + finalhandler: 2.1.1(supports-color@7.2.0) fresh: 2.0.0 http-errors: 2.0.1 merge-descriptors: 2.0.0 @@ -8629,9 +8661,9 @@ snapshots: proxy-addr: 2.0.7 qs: 6.15.1 range-parser: 1.2.1 - router: 2.2.0 - send: 1.2.1 - serve-static: 2.2.1 + router: 2.2.0(supports-color@7.2.0) + send: 1.2.1(supports-color@7.2.0) + serve-static: 2.2.1(supports-color@7.2.0) statuses: 2.0.2 type-is: 2.0.1 vary: 1.1.2 @@ -8709,9 +8741,9 @@ snapshots: dependencies: to-regex-range: 5.0.1 - finalhandler@2.1.1: + finalhandler@2.1.1(supports-color@7.2.0): dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) encodeurl: 2.0.0 escape-html: 1.0.3 on-finished: 2.4.1 @@ -8734,7 +8766,9 @@ snapshots: flatted@3.4.2: {} - follow-redirects@1.16.0: {} + follow-redirects@1.16.0(debug@4.4.3(supports-color@7.2.0)): + optionalDependencies: + debug: 4.4.3(supports-color@7.2.0) for-each@0.3.5: dependencies: @@ -8886,7 +8920,7 @@ snapshots: dependencies: function-bind: 1.1.2 - hast-util-to-jsx-runtime@2.3.6: + hast-util-to-jsx-runtime@2.3.6(supports-color@7.2.0): dependencies: '@types/estree': 1.0.8 '@types/hast': 3.0.4 @@ -8895,9 +8929,9 @@ snapshots: devlop: 1.1.0 estree-util-is-identifier-name: 3.0.0 hast-util-whitespace: 3.0.0 - mdast-util-mdx-expression: 2.0.1 - mdast-util-mdx-jsx: 3.2.0 - mdast-util-mdxjs-esm: 2.0.1 + mdast-util-mdx-expression: 2.0.1(supports-color@7.2.0) + mdast-util-mdx-jsx: 3.2.0(supports-color@7.2.0) + mdast-util-mdxjs-esm: 2.0.1(supports-color@7.2.0) property-information: 7.1.0 space-separated-tokens: 2.0.2 style-to-js: 1.1.21 @@ -8939,17 +8973,17 @@ snapshots: statuses: 2.0.2 toidentifier: 1.0.1 - https-proxy-agent@5.0.1: + https-proxy-agent@5.0.1(supports-color@7.2.0): dependencies: - agent-base: 6.0.2 - debug: 4.4.3 + agent-base: 6.0.2(supports-color@7.2.0) + debug: 4.4.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color - https-proxy-agent@7.0.6: + https-proxy-agent@7.0.6(supports-color@7.2.0): dependencies: agent-base: 7.1.4 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -9398,14 +9432,14 @@ snapshots: unist-util-is: 6.0.1 unist-util-visit-parents: 6.0.2 - mdast-util-from-markdown@2.0.3: + mdast-util-from-markdown@2.0.3(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 '@types/unist': 3.0.3 decode-named-character-reference: 1.3.0 devlop: 1.1.0 mdast-util-to-string: 4.0.0 - micromark: 4.0.2 + micromark: 4.0.2(supports-color@7.2.0) micromark-util-decode-numeric-character-reference: 2.0.2 micromark-util-decode-string: 2.0.1 micromark-util-normalize-identifier: 2.0.1 @@ -9423,67 +9457,67 @@ snapshots: mdast-util-find-and-replace: 3.0.2 micromark-util-character: 2.1.1 - mdast-util-gfm-footnote@2.1.0: + mdast-util-gfm-footnote@2.1.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 micromark-util-normalize-identifier: 2.0.1 transitivePeerDependencies: - supports-color - mdast-util-gfm-strikethrough@2.0.0: + mdast-util-gfm-strikethrough@2.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-gfm-table@2.0.0: + mdast-util-gfm-table@2.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 devlop: 1.1.0 markdown-table: 3.0.4 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-gfm-task-list-item@2.0.0: + mdast-util-gfm-task-list-item@2.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-gfm@3.1.0: + mdast-util-gfm@3.1.0(supports-color@7.2.0): dependencies: - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-gfm-autolink-literal: 2.0.1 - mdast-util-gfm-footnote: 2.1.0 - mdast-util-gfm-strikethrough: 2.0.0 - mdast-util-gfm-table: 2.0.0 - mdast-util-gfm-task-list-item: 2.0.0 + mdast-util-gfm-footnote: 2.1.0(supports-color@7.2.0) + mdast-util-gfm-strikethrough: 2.0.0(supports-color@7.2.0) + mdast-util-gfm-table: 2.0.0(supports-color@7.2.0) + mdast-util-gfm-task-list-item: 2.0.0(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-mdx-expression@2.0.1: + mdast-util-mdx-expression@2.0.1(supports-color@7.2.0): dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-mdx-jsx@3.2.0: + mdast-util-mdx-jsx@3.2.0(supports-color@7.2.0): dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 @@ -9491,7 +9525,7 @@ snapshots: '@types/unist': 3.0.3 ccount: 2.0.1 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 parse-entities: 4.0.2 stringify-entities: 4.0.4 @@ -9500,13 +9534,13 @@ snapshots: transitivePeerDependencies: - supports-color - mdast-util-mdxjs-esm@2.0.1: + mdast-util-mdxjs-esm@2.0.1(supports-color@7.2.0): dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color @@ -9727,10 +9761,10 @@ snapshots: micromark-util-types@2.0.2: {} - micromark@4.0.2: + micromark@4.0.2(supports-color@7.2.0): dependencies: '@types/debug': 4.1.13 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) decode-named-character-reference: 1.3.0 devlop: 1.1.0 micromark-core-commonmark: 2.0.3 @@ -9842,7 +9876,7 @@ snapshots: react: 19.2.4 react-dom: 19.2.4(react@19.2.4) - next@16.2.4(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + next@16.2.4(@babel/core@7.29.0(supports-color@7.2.0))(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4): dependencies: '@next/env': 16.2.4 '@swc/helpers': 0.5.15 @@ -9851,7 +9885,7 @@ snapshots: postcss: 8.4.31 react: 19.2.4 react-dom: 19.2.4(react@19.2.4) - styled-jsx: 5.1.6(@babel/core@7.29.0)(react@19.2.4) + styled-jsx: 5.1.6(@babel/core@7.29.0(supports-color@7.2.0))(react@19.2.4) optionalDependencies: '@next/swc-darwin-arm64': 16.2.4 '@next/swc-darwin-x64': 16.2.4 @@ -10183,17 +10217,17 @@ snapshots: react-is@17.0.2: {} - react-markdown@10.1.0(@types/react@19.2.14)(react@19.2.4): + react-markdown@10.1.0(@types/react@19.2.14)(react@19.2.4)(supports-color@7.2.0): dependencies: '@types/hast': 3.0.4 '@types/mdast': 4.0.4 '@types/react': 19.2.14 devlop: 1.1.0 - hast-util-to-jsx-runtime: 2.3.6 + hast-util-to-jsx-runtime: 2.3.6(supports-color@7.2.0) html-url-attributes: 3.0.1 mdast-util-to-hast: 13.2.1 react: 19.2.4 - remark-parse: 11.0.0 + remark-parse: 11.0.0(supports-color@7.2.0) remark-rehype: 11.1.2 unified: 11.0.5 unist-util-visit: 5.1.0 @@ -10283,21 +10317,21 @@ snapshots: gopd: 1.2.0 set-function-name: 2.0.2 - remark-gfm@4.0.1: + remark-gfm@4.0.1(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 - mdast-util-gfm: 3.1.0 + mdast-util-gfm: 3.1.0(supports-color@7.2.0) micromark-extension-gfm: 3.0.0 - remark-parse: 11.0.0 + remark-parse: 11.0.0(supports-color@7.2.0) remark-stringify: 11.0.0 unified: 11.0.5 transitivePeerDependencies: - supports-color - remark-parse@11.0.0: + remark-parse@11.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) micromark-util-types: 2.0.2 unified: 11.0.5 transitivePeerDependencies: @@ -10366,9 +10400,9 @@ snapshots: '@rolldown/binding-win32-arm64-msvc': 1.0.0-rc.17 '@rolldown/binding-win32-x64-msvc': 1.0.0-rc.17 - router@2.2.0: + router@2.2.0(supports-color@7.2.0): dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) depd: 2.0.0 is-promise: 4.0.0 parseurl: 1.3.3 @@ -10420,9 +10454,9 @@ snapshots: semver@7.7.4: {} - send@1.2.1: + send@1.2.1(supports-color@7.2.0): dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) encodeurl: 2.0.0 escape-html: 1.0.3 etag: 1.8.1 @@ -10436,12 +10470,12 @@ snapshots: transitivePeerDependencies: - supports-color - serve-static@2.2.1: + serve-static@2.2.1(supports-color@7.2.0): dependencies: encodeurl: 2.0.0 escape-html: 1.0.3 parseurl: 1.3.3 - send: 1.2.1 + send: 1.2.1(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -10475,14 +10509,14 @@ snapshots: setprototypeof@1.2.0: {} - shadcn@4.7.0(@types/node@20.19.39)(typescript@6.0.3): + shadcn@4.7.0(@types/node@20.19.39)(supports-color@7.2.0)(typescript@6.0.3): dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/parser': 7.29.3 - '@babel/plugin-transform-typescript': 7.28.6(@babel/core@7.29.0) - '@babel/preset-typescript': 7.28.5(@babel/core@7.29.0) + '@babel/plugin-transform-typescript': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/preset-typescript': 7.28.5(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) '@dotenvx/dotenvx': 1.65.0 - '@modelcontextprotocol/sdk': 1.29.0(zod@3.25.76) + '@modelcontextprotocol/sdk': 1.30.0(supports-color@7.2.0)(zod@3.25.76) '@types/validate-npm-package-name': 4.0.2 browserslist: 4.28.2 commander: 14.0.3 @@ -10494,7 +10528,7 @@ snapshots: fast-glob: 3.3.3 fs-extra: 11.3.4 fuzzysort: 3.1.0 - https-proxy-agent: 7.0.6 + https-proxy-agent: 7.0.6(supports-color@7.2.0) kleur: 4.1.5 msw: 2.14.3(@types/node@20.19.39)(typescript@6.0.3) node-fetch: 3.3.2 @@ -10725,12 +10759,12 @@ snapshots: dependencies: inline-style-parser: 0.2.7 - styled-jsx@5.1.6(@babel/core@7.29.0)(react@19.2.4): + styled-jsx@5.1.6(@babel/core@7.29.0(supports-color@7.2.0))(react@19.2.4): dependencies: client-only: 0.0.1 react: 19.2.4 optionalDependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) supports-color@7.2.0: dependencies: @@ -10873,13 +10907,13 @@ snapshots: possible-typed-array-names: 1.1.0 reflect.getprototypeof: 1.0.10 - typescript-eslint@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3): + typescript-eslint@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3): dependencies: - '@typescript-eslint/eslint-plugin': 8.59.2(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - '@typescript-eslint/typescript-estree': 8.59.2(typescript@6.0.3) - '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - eslint: 9.39.4(jiti@2.7.0) + '@typescript-eslint/eslint-plugin': 8.59.2(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + '@typescript-eslint/typescript-estree': 8.59.2(supports-color@7.2.0)(typescript@6.0.3) + '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -11185,6 +11219,10 @@ snapshots: dependencies: zod: 3.25.76 + zod-to-json-schema@3.25.2(zod@4.4.3): + dependencies: + zod: 4.4.3 + zod-validation-error@4.0.2(zod@4.4.3): dependencies: zod: 4.4.3 diff --git a/scripts/guanxin-case/constants.ts b/scripts/guanxin-case/constants.ts deleted file mode 100644 index d4f6099..0000000 --- a/scripts/guanxin-case/constants.ts +++ /dev/null @@ -1,8 +0,0 @@ -/** - * Dedicated Notebook workspace for the 观心 cardiovascular case. - * workspaces 表没有 title 列，用稳定 userId 标识。 - */ -export const GUANXIN_CASE_USER_ID = "case:guanxin-cardiovascular" - -/** 方案约定的 pilot 规模（20–30）按 sheet×难度 6 层均分。 */ -export const PILOT_CASES_PER_STRATUM = 4 diff --git a/scripts/guanxin-case/ensure-workspace.mts b/scripts/guanxin-case/ensure-workspace.mts deleted file mode 100644 index 963bca6..0000000 --- a/scripts/guanxin-case/ensure-workspace.mts +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Ensure the dedicated 观心 case workspace exists in the Notebook database. - * Requires DATABASE_URL (Notebook Neon/Postgres), not the Knowhere eval DSN. - * - * DATABASE_URL=... node --experimental-strip-types scripts/guanxin-case/ensure-workspace.mts - */ -import { workspaceService } from "../../src/domains/workspace/service.ts" -import { GUANXIN_CASE_USER_ID } from "./constants.ts" - -const workspace = await workspaceService.ensureWorkspace(GUANXIN_CASE_USER_ID) -console.log( - JSON.stringify( - { - userId: workspace.userId, - workspaceId: workspace.id, - namespace: workspace.namespace, - }, - null, - 2, - ), -) diff --git a/scripts/guanxin-case/sample-pilot.py b/scripts/guanxin-case/sample-pilot.py deleted file mode 100644 index a743d4d..0000000 --- a/scripts/guanxin-case/sample-pilot.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -"""Stratified pilot sample from 观心 knowhere自测集.xlsx. - -Takes the first PILOT_CASES_PER_STRATUM rows (by seq_id) from each -(sheet, 难度) bucket. Does not call Knowhere or write the case workspace. -""" -from __future__ import annotations - -import json -import sys -from collections import defaultdict -from pathlib import Path - -import openpyxl - -ROOT = Path(__file__).resolve().parents[2] -DEFAULT_XLSX = Path( - "/Users/wuchengke/Desktop/skills-coding/观心2.0-RAG-v1.1-demo/knowhere自测集.xlsx" -) -DEFAULT_OUT = ROOT / ".tmp" / "guanxin-pilot-cases.json" -PILOT_CASES_PER_STRATUM = 4 - - -def load_rows(xlsx: Path) -> list[dict]: - wb = openpyxl.load_workbook(xlsx, read_only=True, data_only=True) - rows: list[dict] = [] - try: - for sheet_name in wb.sheetnames: - sheet_rows = list(wb[sheet_name].iter_rows(values_only=True)) - if not sheet_rows: - continue - header = [ - str(h).strip() if h is not None else f"col{i}" - for i, h in enumerate(sheet_rows[0]) - ] - for raw in sheet_rows[1:]: - if not raw or raw[0] in (None, ""): - continue - item = { - header[i]: (raw[i] if i < len(raw) else None) - for i in range(len(header)) - } - query = str(item.get("具体query") or "").strip() - if not query: - continue - rows.append( - { - "sheet": sheet_name, - "seq_id": str(item.get("seq_id") or ""), - "query": query, - "disease": str(item.get("具体疾病名称") or "").strip(), - "scene": str(item.get("应用场景-考察能力") or "").strip(), - "difficulty": str(item.get("难度") or "").strip(), - "input_type": str(item.get("输入类型") or "").strip(), - } - ) - finally: - wb.close() - return rows - - -def sample(rows: list[dict]) -> list[dict]: - buckets: dict[tuple[str, str], list[dict]] = defaultdict(list) - for row in rows: - buckets[(row["sheet"], row["difficulty"] or "?")].append(row) - picked: list[dict] = [] - for key in sorted(buckets): - group = sorted(buckets[key], key=lambda row: row["seq_id"]) - picked.extend(group[:PILOT_CASES_PER_STRATUM]) - return sorted(picked, key=lambda row: (row["sheet"], row["seq_id"])) - - -def main() -> None: - xlsx = Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_XLSX - out = Path(sys.argv[2]) if len(sys.argv) > 2 else DEFAULT_OUT - rows = load_rows(xlsx) - picked = sample(rows) - out.parent.mkdir(parents=True, exist_ok=True) - payload = { - "xlsx": str(xlsx), - "per_stratum": PILOT_CASES_PER_STRATUM, - "source_count": len(rows), - "pilot_count": len(picked), - "cases": picked, - } - out.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - print(f"wrote {len(picked)} / {len(rows)} -> {out}") - - -if __name__ == "__main__": - main() diff --git a/src/agent-harness/runtime.test.ts b/src/agent-harness/runtime.test.ts index c2066c3..b459d5b 100644 --- a/src/agent-harness/runtime.test.ts +++ b/src/agent-harness/runtime.test.ts @@ -655,7 +655,6 @@ describe("agent harness runtime", () => { const state: { finalizedManifest?: OutputManifest finalized?: boolean - memorySearchInvoked?: boolean } = {} const tools = createHarnessTools({ state, @@ -671,7 +670,6 @@ describe("agent harness runtime", () => { expect(searchText).toContain('') expect(searchText).toContain('ref="mem:1"') expect(searchText).toContain('itemId="item_1"') - expect(state.memorySearchInvoked).toBe(true) expect(search).toHaveBeenCalledWith({ query: "毛利率", kinds: undefined, @@ -1008,10 +1006,9 @@ describe("agent harness runtime", () => { expect(result.activeTools).not.toContain("knowhere_search") }) - it("keeps Knowhere tools closed for no_retrieval even after memory_search", () => { + it("keeps Knowhere tools closed for no_retrieval", () => { const result = prepareHarnessStep({ stepNumber: 4, - memorySearchInvoked: true, intent: { task: "answer", dependsOnPreviousTurn: false, @@ -1027,8 +1024,8 @@ describe("agent harness runtime", () => { expect(result.activeTools).not.toContain("knowhere_search") }) - it("opens Knowhere tools only after memory_search when sources are required", () => { - const beforeMemory = prepareHarnessStep({ + it("opens memory_search and knowhere_search together as peers when sources are required", () => { + const result = prepareHarnessStep({ stepNumber: 3, intent: { task: "answer", @@ -1040,23 +1037,8 @@ describe("agent harness runtime", () => { }, messages: [], }) - const afterMemory = prepareHarnessStep({ - stepNumber: 4, - memorySearchInvoked: true, - intent: { - task: "answer", - dependsOnPreviousTurn: false, - retrievalNeeded: "yes", - targetModalities: ["text"], - constraints: {}, - groundingPolicy: "must_use_sources", - }, - messages: [], - }) - expect(beforeMemory.activeTools).toContain("memory_search") - expect(beforeMemory.activeTools).not.toContain("knowhere_search") - expect(afterMemory.activeTools).toEqual( + expect(result.activeTools).toEqual( expect.arrayContaining([ "memory_search", "knowhere_search", diff --git a/src/agent-harness/runtime.ts b/src/agent-harness/runtime.ts index 11a6c3a..0f4aab0 100644 --- a/src/agent-harness/runtime.ts +++ b/src/agent-harness/runtime.ts @@ -60,7 +60,6 @@ type HarnessToolState = { inspectedImageRefs?: string[] imageHighlights?: ImageInspectionHighlights[] toolCalls?: HarnessToolCallTrace[] - memorySearchInvoked?: boolean } type HarnessTools = ReturnType @@ -245,7 +244,6 @@ export async function runAgentHarness( messages: stepMessages, stepNumber, intent: state.intent, - memorySearchInvoked: state.memorySearchInvoked === true, hasUninspectedImageAssets: input.inspectImages !== undefined && hasUninspectedImageAssets({ state, ledger }), @@ -299,12 +297,21 @@ const crystalRetrievalTools = [ /** Reserved third retrieval slot (cognition). Not registered this round. */ const cognitionRetrievalTools = [] as const +// TODO(memory-architecture): today the agent itself decides, per turn via +// declareIntent, whether to call memory_search / knowhere_search as MCP +// tools. An alternative considered and deferred: always query Memento +// (including future "cognition") on every turn and let Memento decide what, +// if anything, to inject into context, instead of the agent choosing to call +// a tool. Not adopted now — it would replace this tool-invocation control +// flow with a middleware/auto-inject model and needs its own design + test +// rewrite. Revisit if agent misjudgment on retrieval-needed becomes a real +// problem. + export function prepareHarnessStep(input: { readonly stepNumber: number readonly messages: readonly ModelMessage[] readonly hasUninspectedImageAssets?: boolean readonly intent?: IntentFrame - readonly memorySearchInvoked?: boolean }): HarnessStepPreparation { const messages = sanitizeHarnessModelMessagesForStep(input.messages) @@ -350,14 +357,12 @@ export function prepareHarnessStep(input: { messages, activeTools: selectHarnessActiveTools({ intent: input.intent, - memorySearchInvoked: input.memorySearchInvoked === true, }), } } function selectHarnessActiveTools(input: { readonly intent?: IntentFrame - readonly memorySearchInvoked: boolean }): Array> { const tools: Array> = [ ...alwaysAvailableTools, @@ -366,11 +371,11 @@ function selectHarnessActiveTools(input: { return tools } + // memory_search and knowhere_search are peers: both open together once + // retrieval is allowed. The agent decides which to call and in what + // order — neither tool gates the other. tools.push(...fluidRetrievalTools) - if ( - input.intent?.groundingPolicy === "must_use_sources" && - input.memorySearchInvoked - ) { + if (input.intent?.groundingPolicy === "must_use_sources") { tools.push(...crystalRetrievalTools) } tools.push(...cognitionRetrievalTools) @@ -535,12 +540,10 @@ export function createHarnessTools(input: { toolName: "memory_search", inputSummary: summarizeMemorySearchRequest(request), execute: async () => { - const output = await executeMemorySearch({ + return await executeMemorySearch({ memoryTools: input.memoryTools, request, }) - input.state.memorySearchInvoked = true - return output }, summarizeOutput: summarizeMemoryTextOutput, }), diff --git a/src/app/api/memory/distill/route.ts b/src/app/api/memory/distill/route.ts deleted file mode 100644 index e0230d9..0000000 --- a/src/app/api/memory/distill/route.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { serve } from "@upstash/workflow/nextjs" - -import { - normalizeMemoryDistillPayload, - runMemoryDistillWorkflow, - type MemoryDistillPayload, -} from "@/domains/memory/distill-workflow" -import { logger } from "@/lib/logger" - -export const { POST } = serve( - async (context) => { - const payload = normalizeMemoryDistillPayload(context.requestPayload) - if (!payload) { - logger.warn("memory: distill workflow received invalid payload") - return - } - await runMemoryDistillWorkflow({ context, payload }) - }, - { - failureFunction: async ({ context, failResponse }) => { - logger.error("memory: distill workflow failed", { - payload: context.requestPayload, - failResponse, - }) - }, - }, -) diff --git a/src/app/api/memory/extract/route.ts b/src/app/api/memory/extract/route.ts deleted file mode 100644 index ee25dbe..0000000 --- a/src/app/api/memory/extract/route.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { serve } from "@upstash/workflow/nextjs" - -import { - normalizeMemoryExtractPayload, - runMemoryExtractWorkflow, - type MemoryExtractPayload, -} from "@/domains/memory/extract-workflow" -import { logger } from "@/lib/logger" - -export const { POST } = serve( - async (context) => { - const payload = normalizeMemoryExtractPayload(context.requestPayload) - if (!payload) { - logger.warn("memory: extract workflow received invalid payload") - return - } - await runMemoryExtractWorkflow({ context, payload }) - }, - { - failureFunction: async ({ context, failResponse }) => { - logger.error("memory: extract workflow failed", { - payload: context.requestPayload, - failResponse, - }) - }, - }, -) diff --git a/src/domains/chat/commit-turn.test.ts b/src/domains/chat/commit-turn.test.ts index 62e20b3..97f19da 100644 --- a/src/domains/chat/commit-turn.test.ts +++ b/src/domains/chat/commit-turn.test.ts @@ -3,7 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest" const mocks = vi.hoisted(() => ({ handleChatTurn: vi.fn(), - triggerMemoryExtraction: vi.fn(), + captureMemoryTurn: vi.fn(), recordActivations: vi.fn(), })) @@ -11,14 +11,9 @@ vi.mock("./service", () => ({ handleChatTurn: mocks.handleChatTurn, })) -vi.mock("@/domains/memory/extract-trigger", () => ({ - triggerMemoryExtraction: mocks.triggerMemoryExtraction, -})) - -vi.mock("@/domains/retrieval-activation/service", () => ({ - retrievalActivationService: { - recordActivations: mocks.recordActivations, - }, +vi.mock("@/integrations/memento/client", () => ({ + captureMemoryTurn: mocks.captureMemoryTurn, + recordActivations: mocks.recordActivations, })) import { commitChatTurn } from "./commit-turn" @@ -27,7 +22,8 @@ import type { Workspace } from "@/infrastructure/db/schema" describe("commitChatTurn", () => { beforeEach(() => { vi.clearAllMocks() - mocks.recordActivations.mockResolvedValue(1) + mocks.recordActivations.mockResolvedValue(undefined) + mocks.captureMemoryTurn.mockResolvedValue(undefined) }) it("records fluid memory activations from finalize after a successful turn", async () => { @@ -105,11 +101,12 @@ describe("commitChatTurn", () => { }) expect(Either.isRight(result)).toBe(true) - expect(mocks.triggerMemoryExtraction).toHaveBeenCalledWith({ + expect(mocks.captureMemoryTurn).toHaveBeenCalledWith({ workspaceId: "workspace_1", - threadId: "thread_1", - userMessageId: "msg_user", - assistantMessageId: "msg_assistant", + sourceMessageId: "msg_assistant", + userText: "毛利率", + assistantText: "按已有记忆。", + referencedDocumentIds: ["doc_1"], }) expect(mocks.recordActivations).toHaveBeenCalledWith([ { diff --git a/src/domains/chat/commit-turn.ts b/src/domains/chat/commit-turn.ts index 373bb3a..1113058 100644 --- a/src/domains/chat/commit-turn.ts +++ b/src/domains/chat/commit-turn.ts @@ -1,12 +1,11 @@ import { Either } from "effect" import type { MemoryCitation } from "@/agent-harness" +import { + captureMemoryTurn, + recordActivations, +} from "@/integrations/memento/client" import { generateAgenticOutputManifest } from "./prompt" -import { triggerMemoryExtraction } from "@/domains/memory/extract-trigger" -import { retrievalActivationService } from "@/domains/retrieval-activation/service" -import { toChunkUnitRef } from "@/domains/retrieval-activation/types" -import { summarizeUnknownError } from "@/lib/format-log-value" -import { logger } from "@/lib/logger" import type { ChatCitationView } from "./types" import { handleChatTurn, @@ -17,9 +16,8 @@ import { type CommitChatTurnInput = Parameters[0] /** - * Production chat-turn commit used by the HTTP route and the 观心 batch - * script: answer → persist → extract fluid memory → record cited - * crystal/memory activations. + * Production chat-turn commit used by the HTTP route: answer → persist → + * capture fluid memory on Memento → record cited crystal/memory activations. */ export async function commitChatTurn( input: CommitChatTurnInput, @@ -37,15 +35,19 @@ export async function commitChatTurn( }) if (Either.isRight(result)) { - void triggerMemoryExtraction({ + const [userMessage, assistantMessage] = result.right.messages + void captureMemoryTurn({ workspaceId: input.workspace.id, - threadId: result.right.threadId, - userMessageId: result.right.messages[0].id, - assistantMessageId: result.right.messages[1].id, + sourceMessageId: assistantMessage.id, + userText: userMessage.content, + assistantText: assistantMessage.content, + referencedDocumentIds: collectCitationDocumentIds( + assistantMessage.citations, + ), }) void recordChunkActivations({ workspaceId: input.workspace.id, - citations: result.right.messages[1].citations, + citations: assistantMessage.citations, }) void recordMemoryActivations({ workspaceId: input.workspace.id, @@ -91,17 +93,7 @@ export async function recordChunkActivations(input: { }, ] }) - if (activationInputs.length === 0) return - - try { - await retrievalActivationService.recordActivations(activationInputs) - } catch (error) { - logger.warn("chat: failed to record chunk activations", { - workspaceId: input.workspaceId, - chunkCount: activationInputs.length, - error: summarizeUnknownError(error), - }) - } + await recordActivations(activationInputs) } /** @@ -117,15 +109,25 @@ export async function recordMemoryActivations(input: { unitType: "fluid_memory" as const, unitRef: citation.itemId, })) - if (activationInputs.length === 0) return + await recordActivations(activationInputs) +} - try { - await retrievalActivationService.recordActivations(activationInputs) - } catch (error) { - logger.warn("chat: failed to record memory activations", { - workspaceId: input.workspaceId, - memoryCount: activationInputs.length, - error: summarizeUnknownError(error), - }) +function toChunkUnitRef(input: { + readonly documentId: string + readonly chunkId: string +}): string { + return `${input.documentId}:${input.chunkId}` +} + +function collectCitationDocumentIds( + citations: readonly ChatCitationView[] | undefined, +): string[] { + const ids = new Set() + for (const citation of citations ?? []) { + const documentId = citation.source.documentId + if (typeof documentId === "string" && documentId.length > 0) { + ids.add(documentId) + } } + return [...ids] } diff --git a/src/domains/chat/memory-tools.test.ts b/src/domains/chat/memory-tools.test.ts deleted file mode 100644 index ad083a9..0000000 --- a/src/domains/chat/memory-tools.test.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from "vitest" - -import { DISTILL_DEDUP_CANDIDATES_PER_KIND } from "@/domains/memory/distill-config" -import type { FluidMemoryItem } from "@/infrastructure/db/schema" - -const findDedupCandidates = vi.fn() - -vi.mock("@/domains/memory/service", () => ({ - memoryService: { - findDedupCandidates: (...args: unknown[]) => findDedupCandidates(...args), - }, -})) - -describe("notebookMemoryTools", () => { - beforeEach(() => { - findDedupCandidates.mockReset() - }) - - it("queries all four kinds and assigns mem refs in kind order", async () => { - findDedupCandidates.mockImplementation( - async (_workspaceId: string, kind: string) => { - if (kind === "stance") return [makeMemoryItem({ id: "item_stance" })] - if (kind === "entity_of_interest") { - return [makeMemoryItem({ id: "item_entity", kind: "entity_of_interest" })] - } - return [] - }, - ) - const { notebookMemoryTools } = await import("./memory-tools") - const runtime = notebookMemoryTools.createRuntime({ - workspaceId: "workspace_1", - }) - - const response = await runtime.search({ query: "毛利率 英伟达" }) - - expect(findDedupCandidates).toHaveBeenCalledTimes(4) - expect(findDedupCandidates.mock.calls.map((call) => call[1])).toEqual([ - "indicator_pref", - "stance", - "decision_rule", - "entity_of_interest", - ]) - expect(findDedupCandidates.mock.calls[0]?.[3]).toBe( - DISTILL_DEDUP_CANDIDATES_PER_KIND, - ) - expect(response).toEqual({ - query: "毛利率 英伟达", - items: [ - expect.objectContaining({ - ref: "mem:1", - itemId: "item_stance", - kind: "stance", - }), - expect.objectContaining({ - ref: "mem:2", - itemId: "item_entity", - kind: "entity_of_interest", - }), - ], - }) - }) - - it("searches only the requested kinds", async () => { - findDedupCandidates.mockResolvedValue([]) - const { notebookMemoryTools } = await import("./memory-tools") - const runtime = notebookMemoryTools.createRuntime({ - workspaceId: "workspace_1", - }) - - await runtime.search({ - query: "PE", - kinds: ["indicator_pref"], - }) - - expect(findDedupCandidates).toHaveBeenCalledTimes(1) - expect(findDedupCandidates).toHaveBeenCalledWith( - "workspace_1", - "indicator_pref", - expect.any(Array), - DISTILL_DEDUP_CANDIDATES_PER_KIND, - ) - }) -}) - -function makeMemoryItem( - overrides: Partial = {}, -): FluidMemoryItem { - return { - id: "item_1", - workspaceId: "workspace_1", - kind: "stance", - payload: { statement: "s", scope: "scope", rationale: "r" }, - abstractL0: "abstract", - overviewL1: "overview", - sourceMessageId: null, - confidence: 0.8, - status: "active", - deactivationReason: null, - version: 1, - createdAt: new Date("2026-09-10T00:00:00Z"), - updatedAt: new Date("2026-09-10T00:00:00Z"), - ...overrides, - } -} diff --git a/src/domains/chat/memory-tools.ts b/src/domains/chat/memory-tools.ts deleted file mode 100644 index 63d48b1..0000000 --- a/src/domains/chat/memory-tools.ts +++ /dev/null @@ -1,66 +0,0 @@ -import type { - MemorySearchItem, - MemorySearchRequest, - MemorySearchResponse, - MemoryToolRuntime, -} from "@/agent-harness" -import { DISTILL_DEDUP_CANDIDATES_PER_KIND } from "@/domains/memory/distill-config" -import { tokenizeMemoryText } from "@/domains/memory/search-index" -import { memoryService } from "@/domains/memory/service" -import { fluidMemoryKinds, isFluidMemoryKind } from "@/domains/memory/types" -import type { FluidMemoryItem } from "@/infrastructure/db/schema" - -type NotebookMemoryToolsInput = { - readonly workspaceId: string -} - -export const notebookMemoryTools = { - createRuntime(input: NotebookMemoryToolsInput): MemoryToolRuntime { - return { - search: (request) => searchWorkspaceMemory(input.workspaceId, request), - } - }, -} as const - -async function searchWorkspaceMemory( - workspaceId: string, - request: MemorySearchRequest, -): Promise { - const tokens = tokenizeMemoryText(request.query).map((entry) => entry.token) - const kinds = request.kinds ?? fluidMemoryKinds - const items: MemorySearchItem[] = [] - - for (const kind of kinds) { - const candidates = await memoryService.findDedupCandidates( - workspaceId, - kind, - tokens, - // Same per-kind cap as the existing findDedupCandidates caller. - DISTILL_DEDUP_CANDIDATES_PER_KIND, - ) - for (const candidate of candidates) { - const item = toMemorySearchItem(candidate, items.length + 1) - if (item) items.push(item) - } - } - - return { - query: request.query, - items, - } -} - -function toMemorySearchItem( - item: FluidMemoryItem, - index: number, -): MemorySearchItem | null { - if (!isFluidMemoryKind(item.kind)) return null - - return { - ref: `mem:${index}`, - itemId: item.id, - kind: item.kind, - abstractL0: item.abstractL0, - overviewL1: item.overviewL1, - } -} diff --git a/src/domains/chat/prompt.ts b/src/domains/chat/prompt.ts index d61177e..4f72101 100644 --- a/src/domains/chat/prompt.ts +++ b/src/domains/chat/prompt.ts @@ -16,8 +16,8 @@ import type { ChatHistoryMessage, SearchSources, } from "./contracts" +import { mementoMemoryTools } from "@/integrations/memento/memory-tools" import { notebookKnowhereTools } from "./knowhere-tools" -import { notebookMemoryTools } from "./memory-tools" const RECENT_CONTEXT_MESSAGE_LIMIT = 8 const CONTEXT_CONTENT_CHAR_LIMIT = 900 @@ -65,7 +65,7 @@ export const generateAgenticOutputManifestEffect = ( notebookKnowhereTools.createSearchOnlyRuntime({ searchSources: input.searchSources, }), - memoryTools: notebookMemoryTools.createRuntime({ + memoryTools: mementoMemoryTools.createRuntime({ workspaceId: input.workspaceId, }), ...(input.inspectImages ? { inspectImages: input.inspectImages } : {}), diff --git a/src/domains/chat/route-service.test.ts b/src/domains/chat/route-service.test.ts index 54e8dae..d27b32d 100644 --- a/src/domains/chat/route-service.test.ts +++ b/src/domains/chat/route-service.test.ts @@ -25,7 +25,8 @@ const mocks = vi.hoisted(() => ({ parsedStorageWriteAsset: vi.fn(), softDeleteChatThread: vi.fn(), startBackgroundReconciliation: vi.fn(), - triggerMemoryExtraction: vi.fn(), + captureMemoryTurn: vi.fn(), + recordActivations: vi.fn(), })) vi.mock("ai", async (importOriginal) => { @@ -63,8 +64,9 @@ vi.mock("@/domains/sources/background-reconcile", () => ({ startBackgroundReconciliation: mocks.startBackgroundReconciliation, })) -vi.mock("@/domains/memory/extract-trigger", () => ({ - triggerMemoryExtraction: mocks.triggerMemoryExtraction, +vi.mock("@/integrations/memento/client", () => ({ + captureMemoryTurn: mocks.captureMemoryTurn, + recordActivations: mocks.recordActivations, })) vi.mock("@/domains/sources/workflow-runtime", () => ({ @@ -853,11 +855,12 @@ describe("chat route services", () => { }) expect(result.status).toBe(200) - expect(mocks.triggerMemoryExtraction).toHaveBeenCalledWith({ + expect(mocks.captureMemoryTurn).toHaveBeenCalledWith({ workspaceId: workspace.id, - threadId: "thread_1", - userMessageId: "message_user", - assistantMessageId: "message_assistant", + sourceMessageId: "message_assistant", + userText: "Summarize it", + assistantText: "Summary", + referencedDocumentIds: [], }) expect(mocks.startBackgroundReconciliation).toHaveBeenCalledWith( workspace.id, diff --git a/src/domains/chat/service.test.ts b/src/domains/chat/service.test.ts index 0ac2387..133e0a5 100644 --- a/src/domains/chat/service.test.ts +++ b/src/domains/chat/service.test.ts @@ -83,23 +83,9 @@ describe("handleChatTurn", () => { }); }); - it("allows a turn with no local sources so remote retrieval can still run", async () => { - const retrieval = { - query: vi.fn().mockResolvedValue({ - results: [makeRetrievalResult()], - evidenceText: "Grounding content", - referencedChunks: [], - namespace: "notebook-namespace", - query: "What does the document say?", - routerUsed: "workflow_single_step", - answerText: null, - }), - }; + it("rejects a turn with no local sources without calling retrieval", async () => { + const retrieval = { query: vi.fn() }; const repository = makeRepository(); - const generateAnswer = vi.fn(async ({ searchSources }) => { - await searchSources({ query: "What does the document say?" }); - return makeHarnessRunResult("Grounded answer."); - }); const result = await handleChatTurn({ workspace: makeWorkspace(), @@ -107,13 +93,19 @@ describe("handleChatTurn", () => { question: "What does the document say?", excludedSourceIds: [], retrieval, - generateAnswer, + generateAnswer: vi.fn(), repository, }); - expect(Either.isRight(result)).toBe(true); - expect(generateAnswer).toHaveBeenCalled(); - expect(repository.appendMessageToThread).toHaveBeenCalled(); + expect(Either.isLeft(result)).toBe(true); + if (Either.isLeft(result)) { + expect(result.left).toMatchObject({ + status: 409, + message: "Upload and process a document before asking questions.", + }); + } + expect(retrieval.query).not.toHaveBeenCalled(); + expect(repository.appendMessageToThread).not.toHaveBeenCalled(); }); it("rejects chat before any source is ready without calling retrieval", async () => { diff --git a/src/domains/chat/service.ts b/src/domains/chat/service.ts index b2c5bce..23bfa68 100644 --- a/src/domains/chat/service.ts +++ b/src/domains/chat/service.ts @@ -85,7 +85,7 @@ export const handleChatTurnEffect = (input: ChatTurnInput) => const readySources = input.sources.filter( (source) => source.status === "ready" && source.knowhereDocumentId, ) - if (input.sources.length > 0 && readySources.length === 0) { + if (readySources.length === 0) { return yield* Effect.fail(noReadySources) } diff --git a/src/domains/memory/decay-candidates.test.ts b/src/domains/memory/decay-candidates.test.ts deleted file mode 100644 index 7635c1f..0000000 --- a/src/domains/memory/decay-candidates.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { selectDecayCandidates } from "./decay-candidates" - -const NOW = new Date("2026-01-08T00:00:00Z") -const A_YEAR_AGO = new Date("2025-01-08T00:00:00Z") - -describe("selectDecayCandidates", () => { - it("flags an old, never-activated item below the threshold", () => { - const candidates = selectDecayCandidates({ - items: [{ id: "item_1", kind: "stance", createdAt: A_YEAR_AGO }], - activationsById: new Map(), - now: NOW, - scoreThreshold: 0.3, - }) - - expect(candidates).toEqual([ - { id: "item_1", kind: "stance", score: expect.any(Number), activationCount: 0 }, - ]) - expect(candidates[0]!.score).toBeLessThan(0.3) - }) - - it("does not flag a freshly created item even with no activations", () => { - const candidates = selectDecayCandidates({ - items: [{ id: "item_1", kind: "stance", createdAt: NOW }], - activationsById: new Map(), - now: NOW, - scoreThreshold: 0.3, - }) - - expect(candidates).toEqual([]) - }) - - it("does not flag an old item that was recently activated", () => { - const candidates = selectDecayCandidates({ - items: [{ id: "item_1", kind: "stance", createdAt: A_YEAR_AGO }], - activationsById: new Map([ - ["item_1", { activationCount: 3, lastActivatedAt: NOW }], - ]), - now: NOW, - scoreThreshold: 0.3, - }) - - expect(candidates).toEqual([]) - }) - - it("only flags items strictly below the given threshold", () => { - const items = [ - { id: "item_1", kind: "stance" as const, createdAt: A_YEAR_AGO }, - { id: "item_2", kind: "stance" as const, createdAt: NOW }, - ] - - const noneFlagged = selectDecayCandidates({ - items, - activationsById: new Map(), - now: NOW, - scoreThreshold: 0, - }) - expect(noneFlagged).toEqual([]) - - const allFlagged = selectDecayCandidates({ - items, - activationsById: new Map(), - now: NOW, - scoreThreshold: 1, - }) - expect(allFlagged.map((c) => c.id).sort()).toEqual(["item_1", "item_2"]) - }) -}) diff --git a/src/domains/memory/decay-candidates.ts b/src/domains/memory/decay-candidates.ts deleted file mode 100644 index 159176e..0000000 --- a/src/domains/memory/decay-candidates.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { computeDecayScore } from "@/domains/retrieval-activation/decay-score" - -/** - * Confirmed decay-candidate threshold: at `BASE_HALF_LIFE_DAYS` (14), a - * never-activated item (ceiling 0.5) crosses this after ~24.3 days of - * silence; one activated once after ~60 days; one activated 3x after ~135 - * days. Chosen by simulating `computeDecayScore`'s actual day-counts across - * activation counts and confirming the resulting grace periods, not picked - * a priori — see the `记忆衰减聚类收尾方案` plan. - */ -export const DEFAULT_DECAY_SCORE_THRESHOLD = 0.15 - -export type DecayableItem = { - readonly id: string - /** Raw `fluid_memory_items.kind` column value — carried through, not validated. */ - readonly kind: string - readonly createdAt: Date -} - -export type ItemActivationStats = { - readonly activationCount: number - readonly lastActivatedAt: Date | null -} - -export type DecayCandidate = { - readonly id: string - readonly kind: string - readonly score: number - readonly activationCount: number -} - -/** - * Pure selection: given active fluid_memory items and their (possibly - * absent) activation ledger rows, return the ones whose decay score is - * below `scoreThreshold`. Does not decide the threshold itself and does not - * write anything — per the plan, a decay score crossing the line only - * produces a *candidate*; moving it to `inactive` is a separate, explicit - * step (see `memoryRepository.deactivateDecayedItemsEffect`). - * - * The anchor for an item with no ledger row (never activated) is its own - * `createdAt` — a real, meaningful signal here (unlike a crystal chunk, - * where "no row" means "no signal at all"), so a never-activated item still - * decays normally from the moment it was created. - */ -export function selectDecayCandidates(input: { - readonly items: readonly DecayableItem[] - readonly activationsById: ReadonlyMap - readonly now: Date - readonly scoreThreshold: number -}): readonly DecayCandidate[] { - return input.items.flatMap((item) => { - const activation = input.activationsById.get(item.id) - const activationCount = activation?.activationCount ?? 0 - const anchorAt = activation?.lastActivatedAt ?? item.createdAt - const score = computeDecayScore({ - activationCount, - anchorAt, - now: input.now, - }) - return score < input.scoreThreshold - ? [{ id: item.id, kind: item.kind, score, activationCount }] - : [] - }) -} diff --git a/src/domains/memory/distill-config.ts b/src/domains/memory/distill-config.ts deleted file mode 100644 index 4c82abb..0000000 --- a/src/domains/memory/distill-config.ts +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Distill job defaults from the two-tier fluid-memory plan. - * Tunable later; kept as named constants (not scattered literals). - */ - -/** Process-local cooldown + QStash workflowRunId bucket width (mirrors reconcile). */ -export const DISTILL_COOLDOWN_MS = 5 * 60_000 - -/** Capture may trigger distill once pending observations reach this count. */ -export const DISTILL_MIN_PENDING = 8 - -/** Max pending rows claimed per distill run (oldest first). */ -export const DISTILL_BATCH_MAX = 40 - -/** Lexical dedup candidates loaded per memory kind for one distill batch. */ -export const DISTILL_DEDUP_CANDIDATES_PER_KIND = 8 - -/** Delete consumed observations older than this (retention sweep after distill). */ -export const DISTILL_CONSUMED_RETENTION_MS = 30 * 24 * 60_000 diff --git a/src/domains/memory/distill-model.ts b/src/domains/memory/distill-model.ts deleted file mode 100644 index ef2485f..0000000 --- a/src/domains/memory/distill-model.ts +++ /dev/null @@ -1,83 +0,0 @@ -import "server-only" - -import { generateObject } from "ai" - -import { buildDistillPrompt } from "./distill-prompts" -import { - entityDistillOutputSchema, - experienceDistillOutputSchema, - indicatorDistillOutputSchema, - toMemoryOperations, - type DistillObservationInput, - type DistillPassKind, - type ExistingMemoryContextItem, - type MemoryOperations, -} from "./distill-types" -import { CHAT_MODEL } from "@/lib/ai" -import { summarizeUnknownError } from "@/lib/format-log-value" -import { logger } from "@/lib/logger" - -const MEMORY_EXTRACTION_MODEL = process.env.MEMORY_EXTRACTION_MODEL ?? CHAT_MODEL - -/** - * One structured-output call for a single distill pass. - * Best-effort — distill runs as a background job; a model failure returns - * null so the workflow can skip applying that pass (logged). No multi-level - * fallback chain. - * - * Input: pending observation batch + existing memories of kinds this pass may - * write + allowed document ids for the batch. - * Output: full MemoryOperations with only this pass's arrays populated - * (others empty), ready for resolveMemoryOperations. - */ -export async function distillMemoryPass(input: { - readonly pass: DistillPassKind - readonly workspaceId: string - readonly observations: readonly DistillObservationInput[] - readonly existingItems: readonly ExistingMemoryContextItem[] - readonly referencedDocumentIds: readonly string[] -}): Promise { - const prompt = buildDistillPrompt(input.pass, { - observations: input.observations, - existingItems: input.existingItems, - referencedDocumentIds: input.referencedDocumentIds, - }) - - try { - switch (input.pass) { - case "indicator": { - const response = await generateObject({ - model: MEMORY_EXTRACTION_MODEL, - schema: indicatorDistillOutputSchema, - messages: [{ role: "user", content: prompt }], - }) - return toMemoryOperations("indicator", response.object) - } - case "experience": { - const response = await generateObject({ - model: MEMORY_EXTRACTION_MODEL, - schema: experienceDistillOutputSchema, - messages: [{ role: "user", content: prompt }], - }) - return toMemoryOperations("experience", response.object) - } - case "entity": { - const response = await generateObject({ - model: MEMORY_EXTRACTION_MODEL, - schema: entityDistillOutputSchema, - messages: [{ role: "user", content: prompt }], - }) - return toMemoryOperations("entity", response.object) - } - } - } catch (error) { - logger.warn("memory: distill model call failed; skipping pass", { - workspaceId: input.workspaceId, - pass: input.pass, - model: MEMORY_EXTRACTION_MODEL, - observationCount: input.observations.length, - error: summarizeUnknownError(error), - }) - return null - } -} diff --git a/src/domains/memory/distill-prompts.test.ts b/src/domains/memory/distill-prompts.test.ts deleted file mode 100644 index 9457880..0000000 --- a/src/domains/memory/distill-prompts.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { buildDistillPrompt } from "./distill-prompts" -import type { DistillObservationInput } from "./distill-types" - -const observations: DistillObservationInput[] = [ - { - id: "obs-1", - signal: "看重毛利率", - evidenceQuote: "毛利率是核心", - subjectHint: "毛利率", - confidence: 0.9, - referencedDocumentIds: ["doc-1"], - }, - { - id: "obs-2", - signal: "跟踪英伟达", - evidenceQuote: "英伟达一直在跟踪", - subjectHint: "英伟达", - confidence: 0.8, - referencedDocumentIds: [], - }, -] - -describe("buildDistillPrompt", () => { - it("indicator pass: only indicator schema, no other kind arrays", () => { - const prompt = buildDistillPrompt("indicator", { - observations, - existingItems: [ - { - id: "item-1", - kind: "indicator_pref", - abstractL0: "看重毛利率", - payloadSummary: "毛利率 — 毛利占营收", - }, - ], - referencedDocumentIds: ["doc-1"], - }) - - expect(prompt).toContain("You DISTILL indicator preferences") - expect(prompt).toContain('"indicatorPrefs"') - expect(prompt).not.toContain('"stances"') - expect(prompt).not.toContain('"decisionRules"') - expect(prompt).not.toContain('"entities"') - expect(prompt).toContain("never emit") - expect(prompt).toContain("PENDING OBSERVATIONS") - expect(prompt).toContain("id=obs-1") - expect(prompt).toContain("id=obs-2") - expect(prompt).toContain("看重毛利率") - expect(prompt).toContain("跟踪英伟达") - expect(prompt).toContain("id=item-1") - expect(prompt).toContain("doc-1") - // All observations go to every pass — no kind routing. - expect(prompt.indexOf("obs-1")).toBeLessThan(prompt.indexOf("obs-2")) - }) - - it("experience pass: stances+rules only; still sees full observation batch", () => { - const prompt = buildDistillPrompt("experience", { - observations, - existingItems: [], - referencedDocumentIds: [], - }) - - expect(prompt).toContain("You DISTILL stances and decision rules") - expect(prompt).toContain('"stances"') - expect(prompt).toContain('"decisionRules"') - expect(prompt).not.toContain('"indicatorPrefs"') - expect(prompt).not.toContain('"entities"') - expect(prompt).toContain("id=obs-1") - expect(prompt).toContain("id=obs-2") - expect(prompt).toContain("(no existing memories yet)") - expect(prompt).toContain("(no documents referenced in this batch)") - }) - - it("entity pass: entities only; referenced ids from batch", () => { - const prompt = buildDistillPrompt("entity", { - observations, - existingItems: [ - { - id: "item-4", - kind: "entity_of_interest", - abstractL0: "跟踪英伟达", - payloadSummary: "英伟达 NVDA", - }, - ], - referencedDocumentIds: ["doc-1"], - }) - - expect(prompt).toContain("You DISTILL entities of interest") - expect(prompt).toContain('"entities"') - expect(prompt).not.toContain('"indicatorPrefs"') - expect(prompt).not.toContain('"stances"') - expect(prompt).not.toContain('"decisionRules"') - expect(prompt).toContain("never invent ids") - expect(prompt).toContain("id=item-4") - expect(prompt).toContain("doc-1") - }) - - it("keeps illustrative examples separated from main instructions", () => { - const prompt = buildDistillPrompt("indicator", { - observations: [], - existingItems: [], - referencedDocumentIds: [], - }) - const main = prompt.slice(0, prompt.indexOf("## Illustrative examples")) - expect(main).not.toMatch(/gross margin|市盈率|\bPE\b|Fed|美联储|NVIDIA|英伟达/i) - expect(prompt).toContain("## Illustrative examples (finance vertical") - expect(prompt).toContain("(no pending observations)") - }) -}) diff --git a/src/domains/memory/distill-prompts.ts b/src/domains/memory/distill-prompts.ts deleted file mode 100644 index 3e9e70c..0000000 --- a/src/domains/memory/distill-prompts.ts +++ /dev/null @@ -1,311 +0,0 @@ -import type { - DistillObservationInput, - DistillPassKind, - ExistingMemoryContextItem, -} from "./distill-types" - -/** - * Distill prompts — three isolated passes over the same pending observation - * batch. Each pass sees ALL observations (no kind routing) and only the - * existing memories of kinds that pass may write. - */ - -const INDICATOR_OUTPUT_SCHEMA_BLOCK = `{ - "indicatorPrefs": [{ - "name": "string", - "aliases": ["string"], - "definition": "string", - "polarity": "higher_better|lower_better|context", - "importance": "core|secondary", - "formulaHint": "string (optional — omit if none)", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }] -}` - -const EXPERIENCE_OUTPUT_SCHEMA_BLOCK = `{ - "stances": [{ - "statement": "string (the stance text; do not use a name field)", - "scope": "string", - "rationale": "string", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }], - "decisionRules": [{ - "when": "string", - "then": "string", - "priority": "high|medium|low", - "rationale": "string", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }] -}` - -const ENTITY_OUTPUT_SCHEMA_BLOCK = `{ - "entities": [{ - "name": "string", - "ticker": "string optional", - "aliases": ["string"], - "knowhereDocumentIds": ["only ids listed under REFERENCED DOCUMENT IDS"], - "reason": "string", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }] -}` - -const INDICATOR_ILLUSTRATIVE_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) - -These show shape and judgement only. Distill whatever the observations support; -do not force the batch into this domain or these metric names. - -- Recurring evaluation metric named across clues → one indicatorPref (stable name + definition + polarity + importance). -- Same metric restated with a nuance → merge into the existing item, do not create a second. -- Skip: a one-off number question, document fact, or weak single-mention with no reusable criterion.` - -const EXPERIENCE_ILLUSTRATIVE_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) - -These show shape and judgement only. Distill whatever the observations support; -do not force the batch into this domain. - -- Durable judgement frame (e.g. long-horizon) → one stance (statement + scope + rationale). -- Reusable when → then discipline over the user's criteria → one decisionRule. -- Abstract away one-off instances; keep a single intent per rule. Split unrelated intents. -- Skip: process narration, document facts, or a preference that is only a metric definition (indicators are another pass).` - -const ENTITY_ILLUSTRATIVE_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) - -These show shape and judgement only. Distill whatever the observations support; -do not force the batch into this domain. - -- User actively tracks a named company/issuer across clues → one entity (name + reason; optional ticker/aliases). -- Same subject restated → merge; attach knowhereDocumentIds only from REFERENCED DOCUMENT IDS. -- Skip: a company mentioned only as a one-off fact question, or names that are not subjects of ongoing interest.` - -const INDICATOR_INSTRUCTIONS_BLOCK = `You DISTILL indicator preferences for a user's fluid memory. - -You receive a BATCH of raw observations (cheap clues about what the USER cares -about) plus existing indicator memories. Produce durable indicator_pref items -only. A separate pass handles stances, decision rules, and entities — never emit -those kinds here. - -Constraints: -- One stable topic/name per preference; merge overlapping or synonymous names. -- Capture "what the user repeatedly uses to evaluate", not one-off facts. -- Keep unrelated criteria as separate items; do not mix them into one payload. - -## What to emit - -indicatorPrefs — recurring metrics or criteria the user uses to evaluate things. -Fields: name, aliases, definition, polarity (higher_better | lower_better | context), -importance (core | secondary), optional formulaHint, abstractL0, overviewL1, -confidence, decision. - -## Language - -- Keep this instruction set and enum/field names in English. -- Write every free-text value in the same language the observations use. Do not - translate the user's terms into English unless the observations themselves used English. - -## Decision rules - -- Distill only from the observation batch evidence about the USER. -- Skip document facts, retrieved numbers, and weak/ephemeral clues. -- For every candidate, choose exactly one op against EXISTING MEMORIES: - - create — genuinely new indicator - - skip — already covered, or too weak - - merge — same indicator refined; emit the full merged fields and set targetItemId - - deprecate — user clearly reversed a stored indicator; set targetItemId -- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. -- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when explicitly stated. -- Omit optional fields instead of setting them to null. -- If nothing qualifies, return {"indicatorPrefs": []}.` - -const EXPERIENCE_INSTRUCTIONS_BLOCK = `You DISTILL stances and decision rules (insights) for a user's fluid memory. - -You receive a BATCH of raw observations plus existing stance/decision-rule -memories. Produce durable stances and decisionRules only. A separate pass -handles indicators and entities — never emit those kinds here. - -Constraints: -- Generalizable, reusable insight — not a process log of one session. -- Atomic scope: one intent per decisionRule; split if when would mix goals. -- Abstract away specific one-off entities/ids from the situation framing when the - rule itself is general; keep concrete names only when the insight requires them. -- Do not restate a bare metric definition as a decisionRule — that belongs to the indicator pass. - -## What to emit - -- stances — durable positions that shape how the user weighs evidence. - Fields: statement (required; do not invent a "name" field), scope, rationale, - abstractL0, overviewL1, confidence, decision. -- decisionRules — reusable when → then disciplines the user stated or clearly endorsed. - Fields: when, then, priority (high | medium | low), rationale, abstractL0, - overviewL1, confidence, decision. - -## Language - -- Keep this instruction set and enum/field names in English. -- Write every free-text value in the same language the observations use. Do not - translate the user's terms into English unless the observations themselves used English. - -## Decision rules - -- Distill only from the observation batch evidence about the USER. -- Skip document facts, small talk, and weak/ephemeral clues. -- For every candidate, choose exactly one op against EXISTING MEMORIES: - - create — genuinely new - - skip — already covered, or too weak - - merge — same insight refined; emit the full merged fields and set targetItemId - - deprecate — user clearly reversed a stored item; set targetItemId -- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. -- Prefer one record per insight. Do not invent a near-duplicate decisionRule for a stance that already encodes the same frame unless the user stated an explicit when → then action. -- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when explicitly stated. -- Omit optional fields instead of setting them to null. -- If nothing qualifies, return {"stances": [], "decisionRules": []}.` - -const ENTITY_INSTRUCTIONS_BLOCK = `You DISTILL entities of interest for a user's fluid memory. - -You receive a BATCH of raw observations plus existing entity memories. Produce -durable entity_of_interest items only. A separate pass handles indicators, -stances, and decision rules — never emit those kinds here. - -Constraints: -- Stable card for a subject the USER actively tracks. -- Merge overlapping names/aliases into one item; keep unrelated subjects separate. -- Attach document provenance only from ids listed under REFERENCED DOCUMENT IDS. - -## What to emit - -entities — named subjects the user is actively tracking. -Fields: name, optional ticker, aliases, reason (required), knowhereDocumentIds -(only from REFERENCED DOCUMENT IDS below; never invent ids), abstractL0, -overviewL1, confidence, decision. - -## Language - -- Keep this instruction set and enum/field names in English. -- Write every free-text value in the same language the observations use. Do not - translate the user's terms into English unless the observations themselves used English. - -## Decision rules - -- Distill only from the observation batch evidence about the USER. -- Skip one-off name drops, document facts, and weak/ephemeral mentions. -- For every candidate, choose exactly one op against EXISTING MEMORIES: - - create — genuinely new tracked subject - - skip — already covered, or too weak - - merge — same subject refined; emit the full merged fields and set targetItemId - - deprecate — user clearly stopped tracking / reversed; set targetItemId -- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. -- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when explicitly stated. -- Omit optional fields instead of setting them to null. -- If nothing qualifies, return {"entities": []}.` - -export type BuildDistillPromptInput = { - readonly observations: readonly DistillObservationInput[] - readonly existingItems: readonly ExistingMemoryContextItem[] - readonly referencedDocumentIds: readonly string[] -} - -export function buildDistillPrompt( - pass: DistillPassKind, - input: BuildDistillPromptInput, -): string { - switch (pass) { - case "indicator": - return assemblePrompt({ - instructions: INDICATOR_INSTRUCTIONS_BLOCK, - examples: INDICATOR_ILLUSTRATIVE_BLOCK, - outputSchema: INDICATOR_OUTPUT_SCHEMA_BLOCK, - input, - }) - case "experience": - return assemblePrompt({ - instructions: EXPERIENCE_INSTRUCTIONS_BLOCK, - examples: EXPERIENCE_ILLUSTRATIVE_BLOCK, - outputSchema: EXPERIENCE_OUTPUT_SCHEMA_BLOCK, - input, - }) - case "entity": - return assemblePrompt({ - instructions: ENTITY_INSTRUCTIONS_BLOCK, - examples: ENTITY_ILLUSTRATIVE_BLOCK, - outputSchema: ENTITY_OUTPUT_SCHEMA_BLOCK, - input, - }) - } -} - -function assemblePrompt(args: { - readonly instructions: string - readonly examples: string - readonly outputSchema: string - readonly input: BuildDistillPromptInput -}): string { - const existingBlock = - args.input.existingItems.length === 0 - ? "(no existing memories yet)" - : args.input.existingItems - .map( - (item) => - `- [${item.kind}] id=${item.id} :: ${item.abstractL0} :: ${item.payloadSummary}`, - ) - .join("\n") - - const documentsBlock = - args.input.referencedDocumentIds.length === 0 - ? "(no documents referenced in this batch)" - : args.input.referencedDocumentIds.join(", ") - - const observationsBlock = - args.input.observations.length === 0 - ? "(no pending observations)" - : args.input.observations - .map((observation) => formatObservation(observation)) - .join("\n\n") - - return `${args.instructions} - -${args.examples} - -## Output JSON schema (follow exactly; do not invent fields) - -${args.outputSchema} - -## EXISTING MEMORIES - -${existingBlock} - -## REFERENCED DOCUMENT IDS - -${documentsBlock} - -## PENDING OBSERVATIONS - -${observationsBlock}` -} - -function formatObservation(observation: DistillObservationInput): string { - const subject = - observation.subjectHint && observation.subjectHint.length > 0 - ? observation.subjectHint - : "(none)" - const docs = - observation.referencedDocumentIds.length === 0 - ? "(none)" - : observation.referencedDocumentIds.join(", ") - return `- id=${observation.id} - signal: ${observation.signal} - evidenceQuote: ${observation.evidenceQuote} - subjectHint: ${subject} - confidence: ${observation.confidence} - referencedDocumentIds: ${docs}` -} diff --git a/src/domains/memory/distill-trigger.test.ts b/src/domains/memory/distill-trigger.test.ts deleted file mode 100644 index f4a09d1..0000000 --- a/src/domains/memory/distill-trigger.test.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest" - -import { DISTILL_COOLDOWN_MS, DISTILL_MIN_PENDING } from "./distill-config" - -const mocks = vi.hoisted(() => ({ - loggerError: vi.fn(), - loggerInfo: vi.fn(), - loggerWarn: vi.fn(), - trigger: vi.fn(), - countPendingObservations: vi.fn(), -})) - -vi.mock("@upstash/workflow", () => ({ - Client: class { - trigger = mocks.trigger - }, -})) - -vi.mock("@/lib/logger", () => ({ - logger: { - error: mocks.loggerError, - info: mocks.loggerInfo, - warn: mocks.loggerWarn, - }, -})) - -vi.mock("./service", () => ({ - memoryService: { - countPendingObservations: mocks.countPendingObservations, - }, -})) - -describe("triggerMemoryDistill", () => { - afterEach(async () => { - vi.clearAllMocks() - vi.useRealTimers() - delete process.env.QSTASH_TOKEN - delete process.env.NOTEBOOK_PUBLIC_URL - const { resetMemoryDistillTriggerStateForTests } = await import( - "./distill-trigger" - ) - resetMemoryDistillTriggerStateForTests() - vi.resetModules() - }) - - it("does not trigger when pending is below the plan threshold", async () => { - mocks.countPendingObservations.mockResolvedValue(DISTILL_MIN_PENDING - 1) - process.env.QSTASH_TOKEN = "qstash_token" - - const { triggerMemoryDistill } = await import("./distill-trigger") - await triggerMemoryDistill({ workspaceId: "workspace_1" }) - - expect(mocks.trigger).not.toHaveBeenCalled() - }) - - it("deduplicates workflow triggers only within a bounded cooldown", async () => { - vi.useFakeTimers() - vi.setSystemTime(new Date("2026-06-30T00:00:00.000Z")) - process.env.QSTASH_TOKEN = "qstash_token" - process.env.NOTEBOOK_PUBLIC_URL = "https://notebook.example" - mocks.countPendingObservations.mockResolvedValue(DISTILL_MIN_PENDING) - mocks.trigger.mockResolvedValue({}) - - const { triggerMemoryDistill } = await import("./distill-trigger") - - await triggerMemoryDistill({ workspaceId: "workspace_1" }) - await triggerMemoryDistill({ workspaceId: "workspace_1" }) - - expect(mocks.trigger).toHaveBeenCalledTimes(1) - expect(mocks.trigger).toHaveBeenLastCalledWith({ - url: "https://notebook.example/api/memory/distill", - body: { workspaceId: "workspace_1" }, - workflowRunId: `workspace_1-${Math.floor( - new Date("2026-06-30T00:00:00.000Z").getTime() / DISTILL_COOLDOWN_MS, - )}`, - retries: 3, - }) - - vi.setSystemTime(new Date("2026-06-30T00:05:01.000Z")) - await triggerMemoryDistill({ workspaceId: "workspace_1" }) - - expect(mocks.trigger).toHaveBeenCalledTimes(2) - }) -}) diff --git a/src/domains/memory/distill-trigger.ts b/src/domains/memory/distill-trigger.ts deleted file mode 100644 index a9bd0fd..0000000 --- a/src/domains/memory/distill-trigger.ts +++ /dev/null @@ -1,77 +0,0 @@ -import "server-only" - -import { Client } from "@upstash/workflow" - -import { DISTILL_COOLDOWN_MS, DISTILL_MIN_PENDING } from "./distill-config" -import type { MemoryDistillPayload } from "./distill-workflow" -import { memoryService } from "./service" -import { logger } from "@/lib/logger" - -// Re-trigger protection: process-local cooldown + bucketed workflowRunId. -// Mirrors background-reconcile — same cooldown width keys both guards. - -const lastTriggeredAtByWorkspaceId: Map = new Map() - -function resolveBaseURL(): string { - return process.env.NOTEBOOK_PUBLIC_URL ?? "http://localhost:3000" -} - -/** - * Fire-and-forget distill trigger for one workspace. - * Caller should already know pending may be high; this re-checks the count, - * applies cooldown + bucketed workflowRunId, then enqueues QStash. - */ -export async function triggerMemoryDistill( - payload: MemoryDistillPayload, -): Promise { - const pendingCount = await memoryService.countPendingObservations( - payload.workspaceId, - ) - if (pendingCount < DISTILL_MIN_PENDING) return - - const now = Date.now() - const lastTriggeredAt = lastTriggeredAtByWorkspaceId.get(payload.workspaceId) - if ( - lastTriggeredAt !== undefined && - now - lastTriggeredAt < DISTILL_COOLDOWN_MS - ) { - return - } - lastTriggeredAtByWorkspaceId.set(payload.workspaceId, now) - - const token = process.env.QSTASH_TOKEN - if (!token) { - logger.warn("memory: skipping distill — QSTASH_TOKEN not set", { - workspaceId: payload.workspaceId, - pendingCount, - }) - lastTriggeredAtByWorkspaceId.delete(payload.workspaceId) - return - } - - const url = `${resolveBaseURL()}/api/memory/distill` - try { - await new Client({ token }).trigger({ - url, - body: payload, - workflowRunId: `${payload.workspaceId}-${Math.floor(now / DISTILL_COOLDOWN_MS)}`, - retries: 3, - }) - logger.info("memory: distill workflow triggered", { - workspaceId: payload.workspaceId, - pendingCount, - url, - }) - } catch (error) { - lastTriggeredAtByWorkspaceId.delete(payload.workspaceId) - logger.error("memory: failed to trigger distill workflow", { - workspaceId: payload.workspaceId, - message: error instanceof Error ? error.message : String(error), - }) - } -} - -/** Test helper: clear process-local cooldown map between cases. */ -export function resetMemoryDistillTriggerStateForTests(): void { - lastTriggeredAtByWorkspaceId.clear() -} diff --git a/src/domains/memory/distill-types.test.ts b/src/domains/memory/distill-types.test.ts deleted file mode 100644 index b3b161d..0000000 --- a/src/domains/memory/distill-types.test.ts +++ /dev/null @@ -1,209 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { - entityDistillOutputSchema, - experienceDistillOutputSchema, - indicatorDistillOutputSchema, - memoryOperationsSchema, - summarizePayloadForContext, - toMemoryOperations, -} from "./distill-types" - -describe("distill output schemas", () => { - it("indicator pass accepts prefs and defaults empty array", () => { - expect(indicatorDistillOutputSchema.parse({})).toEqual({ - indicatorPrefs: [], - }) - const parsed = indicatorDistillOutputSchema.parse({ - indicatorPrefs: [ - { - name: "毛利率", - aliases: [], - definition: "毛利占营收", - polarity: "higher_better", - importance: "core", - abstractL0: "看重毛利率", - overviewL1: "用户用毛利率判断质量。", - confidence: 0.9, - decision: { op: "create" }, - }, - ], - }) - expect(parsed.indicatorPrefs).toHaveLength(1) - expect(parsed).not.toHaveProperty("stances") - }) - - it("experience pass accepts stance+rule when required fields are present", () => { - const parsed = experienceDistillOutputSchema.parse({ - stances: [ - { - statement: "长期持有", - scope: "投资 horizon", - rationale: "用户明确说长期", - abstractL0: "长期持有", - overviewL1: "用户以长期视角评估。", - confidence: 1, - decision: { op: "create" }, - }, - ], - decisionRules: [ - { - when: "毛利率连续两季下滑", - then: "减仓观望", - priority: "high", - rationale: "用户自述纪律", - abstractL0: "毛利率下滑则减仓", - overviewL1: "连续两季下滑时减仓观望。", - confidence: 0.95, - decision: { op: "create" }, - }, - ], - }) - expect(parsed.stances[0]?.statement).toBe("长期持有") - expect(parsed.decisionRules).toHaveLength(1) - expect(parsed).not.toHaveProperty("indicatorPrefs") - }) - - it("rejects stance that uses name instead of statement (no coerce)", () => { - expect(() => - experienceDistillOutputSchema.parse({ - stances: [ - { - name: "长期持有", - scope: "投资", - rationale: "用户明确说长期", - abstractL0: "长期持有", - overviewL1: "用户以长期视角评估。", - confidence: 1, - decision: { op: "create" }, - }, - ], - }), - ).toThrow() - }) - - it("rejects entity missing reason (no fill from abstractL0)", () => { - expect(() => - entityDistillOutputSchema.parse({ - entities: [ - { - name: "英伟达", - aliases: ["NVIDIA"], - knowhereDocumentIds: ["doc-1"], - abstractL0: "持续跟踪英伟达", - overviewL1: "用户把英伟达列为跟踪标的。", - confidence: 0.8, - decision: { op: "merge", targetItemId: "item-4" }, - }, - ], - }), - ).toThrow() - }) - - it("entity pass accepts when reason is present", () => { - const parsed = entityDistillOutputSchema.parse({ - entities: [ - { - name: "英伟达", - aliases: ["NVIDIA"], - knowhereDocumentIds: ["doc-1"], - reason: "用户持续跟踪", - abstractL0: "持续跟踪英伟达", - overviewL1: "用户把英伟达列为跟踪标的。", - confidence: 0.8, - decision: { op: "merge", targetItemId: "item-4" }, - }, - ], - }) - expect(parsed.entities[0]?.reason).toBe("用户持续跟踪") - }) - - it("toMemoryOperations expands each pass into the full four-array record", () => { - expect( - toMemoryOperations("indicator", { - indicatorPrefs: [], - }), - ).toEqual({ - indicatorPrefs: [], - stances: [], - decisionRules: [], - entities: [], - }) - - const experience = toMemoryOperations( - "experience", - experienceDistillOutputSchema.parse({ - stances: [ - { - statement: "长期", - scope: "投资", - rationale: "用户说的", - abstractL0: "长期", - overviewL1: "长期视角。", - confidence: 1, - decision: { op: "create" }, - }, - ], - }), - ) - expect(experience.stances).toHaveLength(1) - expect(experience.indicatorPrefs).toEqual([]) - expect(experience.entities).toEqual([]) - - const entity = toMemoryOperations( - "entity", - entityDistillOutputSchema.parse({ - entities: [ - { - name: "英伟达", - aliases: [], - knowhereDocumentIds: [], - reason: "跟踪", - abstractL0: "跟踪英伟达", - overviewL1: "用户跟踪英伟达。", - confidence: 0.7, - decision: { op: "create" }, - }, - ], - }), - ) - expect(entity.entities).toHaveLength(1) - expect(entity.decisionRules).toEqual([]) - }) - - it("memoryOperationsSchema parses the full four-array shape", () => { - const parsed = memoryOperationsSchema.parse({}) - expect(parsed).toEqual({ - indicatorPrefs: [], - stances: [], - decisionRules: [], - entities: [], - }) - }) -}) - -describe("summarizePayloadForContext", () => { - it("summarizes each kind for existing-memory context lines", () => { - expect( - summarizePayloadForContext("indicator_pref", { - name: "毛利率", - definition: "毛利占营收", - }), - ).toBe("毛利率 — 毛利占营收") - expect( - summarizePayloadForContext("stance", { statement: "长期持有" }), - ).toBe("长期持有") - expect( - summarizePayloadForContext("decision_rule", { - when: "下滑", - then: "减仓", - }), - ).toBe("下滑 => 减仓") - expect( - summarizePayloadForContext("entity_of_interest", { - name: "英伟达", - ticker: "NVDA", - }), - ).toBe("英伟达 NVDA") - }) -}) diff --git a/src/domains/memory/distill-types.ts b/src/domains/memory/distill-types.ts deleted file mode 100644 index 44b2274..0000000 --- a/src/domains/memory/distill-types.ts +++ /dev/null @@ -1,190 +0,0 @@ -import { z } from "zod" - -import { - decisionRulePayloadSchema, - entityOfInterestPayloadSchema, - indicatorPreferencePayloadSchema, - stancePayloadSchema, - type FluidMemoryKind, -} from "./types" - -/** - * Distill-pass LLM contracts. - * - * Three separate structured-output schemas (indicator / experience / - * entity). Capture never emits these shapes — distill is the only writer - * of create/merge/deprecate decisions over `fluid_memory_items`. - */ - -function nullToUndefined(value: unknown): unknown { - return value === null ? undefined : value -} - -const decisionSchema = z.object({ - op: z.enum(["create", "skip", "merge", "deprecate"]), - targetItemId: z.preprocess( - nullToUndefined, - z - .string() - .optional() - .describe( - "Required for merge/deprecate: id of the existing memory item. Omit for create/skip.", - ), - ), - reason: z.preprocess( - nullToUndefined, - z - .string() - .optional() - .describe("Short justification, especially for skip/merge/deprecate."), - ), -}) - -const memorySidecarFields = { - abstractL0: z - .string() - .min(1) - .describe("One line, <= 30 words: the essence of this insight."), - overviewL1: z - .string() - .min(1) - .describe("2-3 sentences: what it means and when it applies."), - confidence: z - .number() - .min(0) - .max(1) - .describe("How explicitly the user stated this across the batch (1 = explicit)."), - decision: decisionSchema, -} - -const stanceEntrySchema = stancePayloadSchema.extend(memorySidecarFields) - -const entityEntrySchema = - entityOfInterestPayloadSchema.extend(memorySidecarFields) - -const indicatorEntrySchema = - indicatorPreferencePayloadSchema.extend(memorySidecarFields) - -const decisionRuleEntrySchema = - decisionRulePayloadSchema.extend(memorySidecarFields) - -/** Full four-array shape consumed by resolveMemoryOperations. */ -export const memoryOperationsSchema = z.object({ - indicatorPrefs: z.array(indicatorEntrySchema).default([]), - stances: z.array(stanceEntrySchema).default([]), - decisionRules: z.array(decisionRuleEntrySchema).default([]), - entities: z.array(entityEntrySchema).default([]), -}) - -export type MemoryOperations = z.infer - -/** Pass 1 — indicator preferences only. */ -export const indicatorDistillOutputSchema = z.object({ - indicatorPrefs: z.array(indicatorEntrySchema).default([]), -}) - -/** Pass 2 — stances + decision rules. */ -export const experienceDistillOutputSchema = z.object({ - stances: z.array(stanceEntrySchema).default([]), - decisionRules: z.array(decisionRuleEntrySchema).default([]), -}) - -/** Pass 3 — entities of interest only. */ -export const entityDistillOutputSchema = z.object({ - entities: z.array(entityEntrySchema).default([]), -}) - -export type IndicatorDistillOutput = z.infer -export type ExperienceDistillOutput = z.infer< - typeof experienceDistillOutputSchema -> -export type EntityDistillOutput = z.infer - -export const distillPassKinds = [ - "indicator", - "experience", - "entity", -] as const - -export type DistillPassKind = (typeof distillPassKinds)[number] - -/** Pending observation row shape fed into distill prompts (batch evidence). */ -export type DistillObservationInput = { - readonly id: string - readonly signal: string - readonly evidenceQuote: string - readonly subjectHint: string | null - readonly confidence: number - readonly referencedDocumentIds: readonly string[] -} - -export type ExistingMemoryContextItem = { - readonly id: string - readonly kind: FluidMemoryKind - readonly abstractL0: string - readonly payloadSummary: string -} - -/** Expand a single-pass LLM object into the full MemoryOperations record. */ -export function toMemoryOperations( - pass: DistillPassKind, - output: - | IndicatorDistillOutput - | ExperienceDistillOutput - | EntityDistillOutput, -): MemoryOperations { - switch (pass) { - case "indicator": { - const typed = output as IndicatorDistillOutput - return { - indicatorPrefs: typed.indicatorPrefs, - stances: [], - decisionRules: [], - entities: [], - } - } - case "experience": { - const typed = output as ExperienceDistillOutput - return { - indicatorPrefs: [], - stances: typed.stances, - decisionRules: typed.decisionRules, - entities: [], - } - } - case "entity": { - const typed = output as EntityDistillOutput - return { - indicatorPrefs: [], - stances: [], - decisionRules: [], - entities: typed.entities, - } - } - } -} - -/** Compact payload label for existing-item context in distill prompts. */ -export function summarizePayloadForContext( - kind: FluidMemoryKind, - payload: unknown, -): string { - if (!payload || typeof payload !== "object") return "" - const record = payload as Record - switch (kind) { - case "indicator_pref": - return [record.name, record.definition] - .filter((part) => typeof part === "string" && part.length > 0) - .join(" — ") - case "stance": - return typeof record.statement === "string" ? record.statement : "" - case "decision_rule": - return [record.when, record.then] - .filter((part) => typeof part === "string" && part.length > 0) - .join(" => ") - case "entity_of_interest": - return [record.name, record.ticker] - .filter((part) => typeof part === "string" && part.length > 0) - .join(" ") - } -} diff --git a/src/domains/memory/distill-workflow.test.ts b/src/domains/memory/distill-workflow.test.ts deleted file mode 100644 index a96b576..0000000 --- a/src/domains/memory/distill-workflow.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { - normalizeMemoryDistillPayload, - toDistillObservationInput, -} from "./distill-workflow" -import type { FluidObservation } from "@/infrastructure/db/schema" - -describe("normalizeMemoryDistillPayload", () => { - it("accepts a workspace id", () => { - expect(normalizeMemoryDistillPayload({ workspaceId: "ws-1" })).toEqual({ - workspaceId: "ws-1", - }) - }) - - it("rejects missing or blank workspace id", () => { - expect(normalizeMemoryDistillPayload(null)).toBeNull() - expect(normalizeMemoryDistillPayload({})).toBeNull() - expect(normalizeMemoryDistillPayload({ workspaceId: " " })).toBeNull() - }) -}) - -describe("toDistillObservationInput", () => { - it("maps a pending row without inventing fields", () => { - const row = { - id: "obs-1", - workspaceId: "ws-1", - sourceMessageId: "msg-1", - signal: "看重毛利率", - evidenceQuote: "毛利率是核心", - subjectHint: "毛利率", - referencedDocumentIds: ["doc-1", ""], - confidence: 0.9, - status: "pending", - createdAt: new Date("2026-06-30T00:00:00.000Z"), - consumedAt: null, - } as FluidObservation - - expect(toDistillObservationInput(row)).toEqual({ - id: "obs-1", - signal: "看重毛利率", - evidenceQuote: "毛利率是核心", - subjectHint: "毛利率", - confidence: 0.9, - referencedDocumentIds: ["doc-1"], - }) - }) -}) diff --git a/src/domains/memory/distill-workflow.ts b/src/domains/memory/distill-workflow.ts deleted file mode 100644 index c491a71..0000000 --- a/src/domains/memory/distill-workflow.ts +++ /dev/null @@ -1,231 +0,0 @@ -import "server-only" - -import type { WorkflowContext } from "@upstash/workflow" - -import { - DISTILL_BATCH_MAX, - DISTILL_CONSUMED_RETENTION_MS, - DISTILL_DEDUP_CANDIDATES_PER_KIND, -} from "./distill-config" -import { distillMemoryPass } from "./distill-model" -import { - summarizePayloadForContext, - type DistillObservationInput, - type DistillPassKind, - type ExistingMemoryContextItem, - type MemoryOperations, -} from "./distill-types" -import { resolveMemoryOperations } from "./resolve-operations" -import { tokenizeMemoryText } from "./search-index" -import { memoryService } from "./service" -import type { FluidMemoryKind } from "./types" -import { isFluidMemoryKind } from "./types" -import type { FluidMemoryItem, FluidObservation } from "@/infrastructure/db/schema" -import { logger } from "@/lib/logger" - -export type MemoryDistillPayload = { - readonly workspaceId: string -} - -type MemoryDistillWorkflowContext = Pick< - WorkflowContext, - "run" -> - -const PASS_KINDS: readonly { - readonly pass: DistillPassKind - readonly kinds: readonly FluidMemoryKind[] -}[] = [ - { pass: "indicator", kinds: ["indicator_pref"] }, - { pass: "experience", kinds: ["stance", "decision_rule"] }, - { pass: "entity", kinds: ["entity_of_interest"] }, -] - -export function normalizeMemoryDistillPayload( - raw: unknown, -): MemoryDistillPayload | null { - if (!raw || typeof raw !== "object") return null - const workspaceId = getNonEmptyString( - (raw as Record).workspaceId, - ) - if (!workspaceId) return null - return { workspaceId } -} - -/** - * Periodic distill: pending observations → three typed passes → resolve → - * write fluid_memory_items and mark the batch consumed. Capture never writes - * the permanent layer; this job is the only writer. - */ -export async function runMemoryDistillWorkflow(input: { - readonly context: MemoryDistillWorkflowContext - readonly payload: MemoryDistillPayload -}): Promise { - const { context, payload } = input - - const batch = await context.run("select-batch", () => - memoryService.listPendingObservations( - payload.workspaceId, - DISTILL_BATCH_MAX, - ), - ) - if (batch.length === 0) { - logger.info("memory: distill skipped — no pending observations", { - workspaceId: payload.workspaceId, - }) - return - } - - const observationInputs = batch.map(toDistillObservationInput) - const referencedDocumentIds = unionDocumentIds(batch) - const queryTokens = tokenizeBatch(observationInputs) - - const candidatesByKind = await context.run("load-candidates", async () => { - const result: Partial> = {} - for (const kind of [ - "indicator_pref", - "stance", - "decision_rule", - "entity_of_interest", - ] as const) { - result[kind] = await memoryService.findDedupCandidates( - payload.workspaceId, - kind, - queryTokens, - DISTILL_DEDUP_CANDIDATES_PER_KIND, - ) - } - return result - }) - - const passOperations: MemoryOperations[] = [] - for (const { pass, kinds } of PASS_KINDS) { - const existingItems = kinds.flatMap((kind) => - (candidatesByKind[kind] ?? []).flatMap((item) => { - const mapped = toExistingMemoryContextItem(item) - return mapped ? [mapped] : [] - }), - ) - const operations = await context.run(`distill-${pass}`, () => - distillMemoryPass({ - pass, - workspaceId: payload.workspaceId, - observations: observationInputs, - existingItems, - referencedDocumentIds, - }), - ) - // Null = model failure for this pass only; other passes still apply. - if (operations) passOperations.push(operations) - } - - if (passOperations.length === 0) { - logger.warn( - "memory: distill aborted — all passes failed; batch left pending", - { - workspaceId: payload.workspaceId, - batchSize: batch.length, - }, - ) - return - } - - const existingItemRefs = Object.values(candidatesByKind) - .flat() - .map((item) => ({ - id: item.id, - kind: item.kind, - status: item.status, - payload: item.payload, - })) - - const resolved = passOperations.flatMap((operations) => - resolveMemoryOperations({ - operations, - existingItems: existingItemRefs, - referencedDocumentIds, - }), - ) - - const applied = await context.run("apply-and-consume", () => - memoryService.applyDistillBatch({ - workspaceId: payload.workspaceId, - sourceMessageId: null, - operations: resolved, - observationIds: batch.map((row) => row.id), - }), - ) - - const deleted = await context.run("retention", () => - memoryService.deleteExpiredConsumedObservations( - new Date(Date.now() - DISTILL_CONSUMED_RETENTION_MS), - ), - ) - - logger.info("memory: distill workflow finished", { - workspaceId: payload.workspaceId, - batchSize: batch.length, - resolvedCount: resolved.length, - diffCount: applied.diffs.length, - consumedCount: applied.consumedCount, - retentionDeleted: deleted, - }) -} - -export function toDistillObservationInput( - row: FluidObservation, -): DistillObservationInput { - const documentIds = Array.isArray(row.referencedDocumentIds) - ? row.referencedDocumentIds.filter( - (id): id is string => typeof id === "string" && id.length > 0, - ) - : [] - return { - id: row.id, - signal: row.signal, - evidenceQuote: row.evidenceQuote, - subjectHint: row.subjectHint, - confidence: row.confidence, - referencedDocumentIds: documentIds, - } -} - -function toExistingMemoryContextItem( - item: FluidMemoryItem, -): ExistingMemoryContextItem | null { - if (!isFluidMemoryKind(item.kind)) return null - return { - id: item.id, - kind: item.kind, - abstractL0: item.abstractL0, - payloadSummary: summarizePayloadForContext(item.kind, item.payload), - } -} - -function tokenizeBatch( - observations: readonly DistillObservationInput[], -): string[] { - const text = observations - .map((observation) => - [observation.signal, observation.subjectHint ?? "", observation.evidenceQuote] - .filter((part) => part.length > 0) - .join(" "), - ) - .join(" ") - return tokenizeMemoryText(text).map((entry) => entry.token) -} - -function unionDocumentIds(rows: readonly FluidObservation[]): string[] { - const ids = new Set() - for (const row of rows) { - if (!Array.isArray(row.referencedDocumentIds)) continue - for (const id of row.referencedDocumentIds) { - if (typeof id === "string" && id.length > 0) ids.add(id) - } - } - return [...ids] -} - -function getNonEmptyString(value: unknown): string | null { - return typeof value === "string" && value.trim().length > 0 ? value : null -} diff --git a/src/domains/memory/extract-trigger.ts b/src/domains/memory/extract-trigger.ts deleted file mode 100644 index 35b6467..0000000 --- a/src/domains/memory/extract-trigger.ts +++ /dev/null @@ -1,43 +0,0 @@ -import "server-only" - -import { Client } from "@upstash/workflow" - -import type { MemoryExtractPayload } from "./extract-workflow" -import { logger } from "@/lib/logger" - -function resolveBaseURL(): string { - return process.env.NOTEBOOK_PUBLIC_URL ?? "http://localhost:3000" -} - -/** - * Fire-and-forget trigger for the post-turn fluid-memory extraction - * workflow. Each turn is uniquely keyed by its message ids, so unlike the - * source reconcile trigger no cooldown/dedup guard is needed; QStash - * retries cover transient delivery failures. - */ -export async function triggerMemoryExtraction( - payload: MemoryExtractPayload, -): Promise { - const token = process.env.QSTASH_TOKEN - if (!token) { - logger.warn("memory: skipping extraction — QSTASH_TOKEN not set", { - workspaceId: payload.workspaceId, - assistantMessageId: payload.assistantMessageId, - }) - return - } - - try { - await new Client({ token }).trigger({ - url: `${resolveBaseURL()}/api/memory/extract`, - body: payload, - retries: 3, - }) - } catch (error) { - logger.error("memory: failed to trigger extraction workflow", { - workspaceId: payload.workspaceId, - assistantMessageId: payload.assistantMessageId, - message: error instanceof Error ? error.message : String(error), - }) - } -} diff --git a/src/domains/memory/extract-workflow.test.ts b/src/domains/memory/extract-workflow.test.ts deleted file mode 100644 index 41205a5..0000000 --- a/src/domains/memory/extract-workflow.test.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { normalizeMemoryExtractPayload } from "./extract-workflow" - -describe("normalizeMemoryExtractPayload", () => { - it("accepts a complete payload", () => { - expect( - normalizeMemoryExtractPayload({ - workspaceId: "ws-1", - threadId: "th-1", - userMessageId: "u-1", - assistantMessageId: "a-1", - }), - ).toEqual({ - workspaceId: "ws-1", - threadId: "th-1", - userMessageId: "u-1", - assistantMessageId: "a-1", - }) - }) - - it("rejects missing or blank fields", () => { - expect(normalizeMemoryExtractPayload(null)).toBeNull() - expect( - normalizeMemoryExtractPayload({ - workspaceId: "ws-1", - threadId: "th-1", - userMessageId: "u-1", - }), - ).toBeNull() - expect( - normalizeMemoryExtractPayload({ - workspaceId: " ", - threadId: "th-1", - userMessageId: "u-1", - assistantMessageId: "a-1", - }), - ).toBeNull() - }) -}) diff --git a/src/domains/memory/extract-workflow.ts b/src/domains/memory/extract-workflow.ts deleted file mode 100644 index ec0a538..0000000 --- a/src/domains/memory/extract-workflow.ts +++ /dev/null @@ -1,128 +0,0 @@ -import "server-only" - -import type { WorkflowContext } from "@upstash/workflow" - -import { triggerMemoryDistill } from "./distill-trigger" -import { captureObservations } from "./extraction-model" -import { memoryService } from "./service" -import { chatThreadService } from "@/domains/chat/thread-service" -import { logger } from "@/lib/logger" - -export type MemoryExtractPayload = { - readonly workspaceId: string - readonly threadId: string - readonly userMessageId: string - readonly assistantMessageId: string -} - -type MemoryExtractWorkflowContext = Pick< - WorkflowContext, - "run" -> - -export function normalizeMemoryExtractPayload( - raw: unknown, -): MemoryExtractPayload | null { - if (!raw || typeof raw !== "object") return null - const record = raw as Record - const workspaceId = getNonEmptyString(record.workspaceId) - const threadId = getNonEmptyString(record.threadId) - const userMessageId = getNonEmptyString(record.userMessageId) - const assistantMessageId = getNonEmptyString(record.assistantMessageId) - if (!workspaceId || !threadId || !userMessageId || !assistantMessageId) { - return null - } - return { workspaceId, threadId, userMessageId, assistantMessageId } -} - -/** - * Per-turn coarse capture: load the turn → LLM observations → append-only - * insert into fluid_observations. Never writes fluid_memory_items (distill - * owns that). After persist, maybe-trigger distill when pending is high enough. - */ -export async function runMemoryExtractWorkflow(input: { - readonly context: MemoryExtractWorkflowContext - readonly payload: MemoryExtractPayload -}): Promise { - const { context, payload } = input - - const turn = await context.run("load-turn", async () => { - const messages = await chatThreadService.listMessages( - payload.workspaceId, - payload.threadId, - ) - const userMessage = messages?.find( - (message) => message.id === payload.userMessageId, - ) - const assistantMessage = messages?.find( - (message) => message.id === payload.assistantMessageId, - ) - if (!userMessage || !assistantMessage) return null - return { - userText: userMessage.content, - assistantText: assistantMessage.content, - referencedDocumentIds: collectCitationDocumentIds( - assistantMessage.citations, - ), - } - }) - if (!turn) { - logger.warn("memory: capture skipped — turn messages not found", { - workspaceId: payload.workspaceId, - threadId: payload.threadId, - }) - return - } - - const observations = await context.run("capture", () => - captureObservations({ - workspaceId: payload.workspaceId, - userText: turn.userText, - assistantText: turn.assistantText, - referencedDocumentIds: turn.referencedDocumentIds, - }), - ) - if (!observations) return - - const inserted = await context.run("persist-observations", async () => { - if (observations.length === 0) return [] - return memoryService.insertObservations({ - workspaceId: payload.workspaceId, - sourceMessageId: payload.assistantMessageId, - referencedDocumentIds: turn.referencedDocumentIds, - observations, - }) - }) - - await context.run("maybe-trigger-distill", async () => { - await triggerMemoryDistill({ workspaceId: payload.workspaceId }) - }) - - logger.info("memory: capture workflow finished", { - workspaceId: payload.workspaceId, - threadId: payload.threadId, - assistantMessageId: payload.assistantMessageId, - observationCount: inserted.length, - }) -} - -function collectCitationDocumentIds(citations: unknown): string[] { - if (!Array.isArray(citations)) return [] - const ids = new Set() - for (const citation of citations) { - if (!citation || typeof citation !== "object") continue - const source = (citation as Record).source - if (!source || typeof source !== "object") continue - const documentId = (source as Record).documentId - if (typeof documentId === "string" && documentId.length > 0) { - ids.add(documentId) - } - } - return [...ids] -} - -function getNonEmptyString(value: unknown): string | null { - return typeof value === "string" && value.trim().length > 0 - ? value - : null -} diff --git a/src/domains/memory/extraction-model.ts b/src/domains/memory/extraction-model.ts deleted file mode 100644 index e013e36..0000000 --- a/src/domains/memory/extraction-model.ts +++ /dev/null @@ -1,50 +0,0 @@ -import "server-only" - -import { generateObject } from "ai" - -import { - captureOutputSchema, - type CaptureOutput, - type CapturedObservation, -} from "./observation-types" -import { buildCapturePrompt } from "./prompts" -import { CHAT_MODEL } from "@/lib/ai" -import { summarizeUnknownError } from "@/lib/format-log-value" -import { logger } from "@/lib/logger" - -const MEMORY_EXTRACTION_MODEL = process.env.MEMORY_EXTRACTION_MODEL ?? CHAT_MODEL - -/** - * One structured-output call: conversation turn in, raw observations out. - * Best-effort — this runs as a background job, so a model failure skips the - * turn (logged) instead of degrading through fallbacks; the clue typically - * resurfaces in a later turn. - */ -export async function captureObservations(input: { - readonly workspaceId: string - readonly userText: string - readonly assistantText: string - readonly referencedDocumentIds: readonly string[] -}): Promise { - try { - const response = await generateObject({ - model: MEMORY_EXTRACTION_MODEL, - schema: captureOutputSchema, - messages: [ - { - role: "user", - content: buildCapturePrompt(input), - }, - ], - }) - const output: CaptureOutput = response.object - return output.observations - } catch (error) { - logger.warn("memory: capture model call failed; skipping turn", { - workspaceId: input.workspaceId, - model: MEMORY_EXTRACTION_MODEL, - error: summarizeUnknownError(error), - }) - return null - } -} diff --git a/src/domains/memory/observation-types.test.ts b/src/domains/memory/observation-types.test.ts deleted file mode 100644 index 0a99eb6..0000000 --- a/src/domains/memory/observation-types.test.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { - captureOutputSchema, - capturedObservationSchema, -} from "./observation-types" - -describe("capturedObservationSchema", () => { - it("accepts a full observation", () => { - const parsed = capturedObservationSchema.parse({ - signal: "看重毛利率", - evidenceQuote: "毛利率是核心", - subjectHint: "毛利率", - confidence: 0.9, - }) - expect(parsed.subjectHint).toBe("毛利率") - expect(parsed).not.toHaveProperty("kindHint") - }) - - it("coerces null subjectHint to undefined (single preprocess)", () => { - const parsed = capturedObservationSchema.parse({ - signal: "长期持有", - evidenceQuote: "我做长期投资", - subjectHint: null, - confidence: 1, - }) - expect(parsed.subjectHint).toBeUndefined() - }) - - it("strips unknown kindHint if the model still emits it", () => { - const parsed = capturedObservationSchema.parse({ - signal: "看重毛利率", - evidenceQuote: "毛利率是核心", - kindHint: "indicator", - confidence: 0.9, - }) - expect(parsed).not.toHaveProperty("kindHint") - }) - - it("rejects empty signal or evidenceQuote", () => { - expect(() => - capturedObservationSchema.parse({ - signal: "", - evidenceQuote: "x", - confidence: 0.5, - }), - ).toThrow() - expect(() => - capturedObservationSchema.parse({ - signal: "x", - evidenceQuote: "", - confidence: 0.5, - }), - ).toThrow() - }) -}) - -describe("captureOutputSchema", () => { - it("defaults missing observations to empty array", () => { - expect(captureOutputSchema.parse({})).toEqual({ observations: [] }) - }) - - it("parses a batch of observations", () => { - const parsed = captureOutputSchema.parse({ - observations: [ - { - signal: "跟踪英伟达", - evidenceQuote: "英伟达一直在跟踪", - subjectHint: "英伟达", - confidence: 0.8, - }, - ], - }) - expect(parsed.observations).toHaveLength(1) - expect(parsed.observations[0]?.subjectHint).toBe("英伟达") - expect(parsed.observations[0]).not.toHaveProperty("kindHint") - }) -}) diff --git a/src/domains/memory/observation-types.ts b/src/domains/memory/observation-types.ts deleted file mode 100644 index 63f7598..0000000 --- a/src/domains/memory/observation-types.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { z } from "zod" - -/** - * Coarse-capture contract for the raw observation layer. - * - * Capture records durable USER clues (what the user cares about) only. - * It does not classify into final memory kinds, does not dedup, and never - * writes `fluid_memory_items`. `subjectHint` is an optional topic anchor - * for later clustering — distill owns authoritative typing and merge. - */ - -/** Single concentrated null→undefined coerce for optional capture fields. */ -function nullToUndefined(value: unknown): unknown { - return value === null ? undefined : value -} - -export const capturedObservationSchema = z.object({ - signal: z - .string() - .min(1) - .describe( - "One durable clue about what the USER cares about, in the user's language.", - ), - evidenceQuote: z - .string() - .min(1) - .describe("Short verbatim snippet from the USER turn that supports signal."), - subjectHint: z.preprocess( - nullToUndefined, - z - .string() - .min(1) - .optional() - .describe( - "Optional short topic anchor (metric name, company, topic). Prefer omit when none.", - ), - ), - confidence: z - .number() - .min(0) - .max(1) - .describe("How explicitly the user stated this (1 = explicit)."), -}) - -export const captureOutputSchema = z.object({ - observations: z.array(capturedObservationSchema).default([]), -}) - -export type CapturedObservation = z.infer -export type CaptureOutput = z.infer diff --git a/src/domains/memory/prompts.test.ts b/src/domains/memory/prompts.test.ts deleted file mode 100644 index 2f7c83f..0000000 --- a/src/domains/memory/prompts.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { buildCapturePrompt } from "./prompts" - -describe("buildCapturePrompt", () => { - const prompt = buildCapturePrompt({ - userText: "毛利率是核心。", - assistantText: "明白。", - referencedDocumentIds: ["doc-1"], - }) - - it("keeps main instructions domain-agnostic and capture-only", () => { - const main = prompt.slice(0, prompt.indexOf("## Illustrative examples")) - expect(main).not.toMatch(/gross margin|市盈率|\bPE\b|Fed|美联储|NVIDIA|英伟达/i) - expect(main).toContain("RAW OBSERVATIONS") - expect(main).toContain("Do NOT classify observations into those kinds") - expect(main).toContain("Do NOT emit any") - expect(main).toContain("kind / type / category field") - expect(main).toContain("Do NOT invent") - expect(main).toContain("create/merge/deprecate operations") - expect(main).not.toContain("kindHint") - expect(main).not.toContain("EXISTING MEMORIES") - expect(main).not.toContain("indicatorPrefs") - expect(main).not.toContain("decisionRules") - expect(main).toContain("Write every free-text value") - expect(main).toMatch(/same\s+language the USER wrote/) - }) - - it("output schema has no early kind classification field", () => { - const schema = prompt.slice(prompt.indexOf("## Output JSON schema")) - expect(schema).not.toContain("kindHint") - expect(schema).toContain("subjectHint") - expect(schema).toContain("signal") - expect(schema).toContain("evidenceQuote") - expect(schema).toContain("confidence") - }) - - it("keeps illustrative examples in a separate section", () => { - expect(prompt).toContain("## Illustrative examples (finance vertical") - expect(prompt).toContain("not exhaustive, not required vocabulary") - const examples = prompt.slice( - prompt.indexOf("## Illustrative examples"), - prompt.indexOf("## Output JSON schema"), - ) - expect(examples).toContain("finance vertical") - expect(examples).toContain("do not force the conversation into this domain") - }) - - it("injects turn context and referenced docs; no existing-memory block", () => { - expect(prompt).toContain("[user]\n毛利率是核心。") - expect(prompt).toContain("doc-1") - expect(prompt).not.toContain("## EXISTING MEMORIES") - }) -}) diff --git a/src/domains/memory/prompts.ts b/src/domains/memory/prompts.ts deleted file mode 100644 index 2f12965..0000000 --- a/src/domains/memory/prompts.ts +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Coarse-capture prompt for the raw observation layer. - * - * Capture extracts durable USER points of concern only. It does not classify - * into final memory kinds, does not dedup against existing items, and does not - * emit create/merge/deprecate decisions — those belong to distill. - */ - -/** Structural output shape only — no domain content. */ -const CAPTURE_OUTPUT_SCHEMA_BLOCK = `{ - "observations": [{ - "signal": "string — one durable clue about what the USER cares about", - "evidenceQuote": "string — short verbatim USER snippet supporting signal", - "subjectHint": "string optional — topic anchor (metric name / company / topic)", - "confidence": 0.0 - }] -}` - -/** - * Illustrative only — kept separate so the model does not treat these domain - * phrases as required vocabulary. - */ -const ILLUSTRATIVE_EXAMPLES_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) - -These show shape and judgement only. Capture whatever the user actually said; -do not force the conversation into this domain. - -- User names a recurring evaluation metric → one observation (signal + quote). -- User states a durable judgement frame (e.g. long-horizon) → one observation. -- User states a reusable when → then discipline → one observation. -- User says they actively track a named company → one observation. -- Skip: a one-off factual question about a page/number, small talk, or an - assistant suggestion the user did not endorse.` - -const MAIN_INSTRUCTIONS_BLOCK = `You capture RAW OBSERVATIONS for a user's fluid memory pipeline. - -These are cheap, high-recall clues about what the USER cares about. A later -distill step will decide final kinds (indicator / rule / stance / entity) and -merge them. Do NOT classify observations into those kinds. Do NOT emit any -kind / type / category field. Do NOT deduplicate. Do NOT invent -create/merge/deprecate operations. - -Document facts live elsewhere (crystal memory). Never capture document facts, -retrieved numbers, or page content as observations. - -## What to capture - -From the USER turn only, emit zero or more observations when there is real -evidence of a durable, reusable point of concern: - -- signal — one short clue in the user's language (what to remember later). -- evidenceQuote — a short verbatim snippet from the USER turn that supports it. -- subjectHint — optional short topic anchor (metric name, company, topic). Prefer omit when none. -- confidence — 1 only when the user stated it explicitly. - -## Language - -- Keep this instruction set and enum/field names in English. -- Write every free-text value (signal, evidenceQuote, subjectHint) in the same - language the USER wrote in this turn. Do not translate the user's terms into - English unless the user themselves used English. - -## Judgement - -- Capture only durable, reusable clues about what the USER cares about. -- Skip one-off questions, document facts, small talk, and assistant claims the - user did not endorse. -- Prefer atomic clues: one observation per distinct clue. Do not merge unrelated - ideas into one signal. -- Omit optional fields instead of setting them to null. -- If nothing is worth capturing, return {"observations": []}.` - -export function buildCapturePrompt(input: { - readonly userText: string - readonly assistantText: string - readonly referencedDocumentIds: readonly string[] -}): string { - const documentsBlock = - input.referencedDocumentIds.length === 0 - ? "(no documents referenced in this turn)" - : input.referencedDocumentIds.join(", ") - - return `${MAIN_INSTRUCTIONS_BLOCK} - -${ILLUSTRATIVE_EXAMPLES_BLOCK} - -## Output JSON schema (follow exactly; do not invent fields) - -${CAPTURE_OUTPUT_SCHEMA_BLOCK} - -## REFERENCED DOCUMENT IDS - -${documentsBlock} - -## CONVERSATION TURN - -[user] -${input.userText} - -[assistant] -${input.assistantText}` -} diff --git a/src/domains/memory/repository.ts b/src/domains/memory/repository.ts deleted file mode 100644 index 9b44f87..0000000 --- a/src/domains/memory/repository.ts +++ /dev/null @@ -1,500 +0,0 @@ -import "server-only" - -import { and, asc, count, eq, inArray, lt, sql } from "drizzle-orm" -import { Effect } from "effect" - -import type { CapturedObservation } from "./observation-types" -import { toDiffOperation, type ResolvedMemoryOperation } from "./resolve-operations" -import { buildMemoryItemTokens } from "./search-index" -import type { - FluidMemoryDeactivationReason, - FluidMemoryKind, - FluidMemoryPayload, - MemoryDiffOperation, -} from "./types" -import { DbClient, type Db } from "@/infrastructure/db" -import { - fluidMemoryItems, - fluidMemoryTokens, - fluidObservations, - memoryDiffs, - type FluidMemoryItem, - type FluidObservation, - type NewFluidMemoryToken, -} from "@/infrastructure/db/schema" - -export type InsertObservationsInput = { - readonly workspaceId: string - readonly sourceMessageId: string | null - readonly referencedDocumentIds: readonly string[] - readonly observations: readonly CapturedObservation[] -} - -export type ApplyDistillBatchInput = { - readonly workspaceId: string - readonly sourceMessageId: string | null - readonly operations: readonly ResolvedMemoryOperation[] - readonly observationIds: readonly string[] -} - -type MemoryRepository = { - readonly findDedupCandidatesEffect: ( - workspaceId: string, - kind: FluidMemoryKind, - tokens: readonly string[], - limit: number, - ) => Effect.Effect - readonly insertObservationsEffect: ( - input: InsertObservationsInput, - ) => Effect.Effect - readonly countPendingObservationsEffect: ( - workspaceId: string, - ) => Effect.Effect - readonly listPendingObservationsEffect: ( - workspaceId: string, - limit: number, - ) => Effect.Effect - readonly applyDistillBatchEffect: ( - input: ApplyDistillBatchInput, - ) => Effect.Effect< - { - readonly diffs: readonly MemoryDiffOperation[] - readonly consumedCount: number - }, - never, - DbClient - > - readonly deleteExpiredConsumedObservationsEffect: ( - olderThan: Date, - ) => Effect.Effect - readonly listActiveItemsEffect: ( - workspaceId: string, - ) => Effect.Effect< - readonly Pick[], - never, - DbClient - > - /** - * Move a specific set of active items to `inactive` with reason - * `decayed` (the activation-decay job's candidates, already confirmed by - * the caller — this never decides which items on its own). Mirrors the - * distill `deprecate` write path: drops the item's token rows so it stops - * surfacing as a dedup candidate. - */ - readonly deactivateDecayedItemsEffect: ( - workspaceId: string, - itemIds: readonly string[], - ) => Effect.Effect -} - -type RawRowsResult = readonly Row[] | { readonly rows: readonly Row[] } - -type TxClient = Parameters[0]>[0] - -/** - * Retrieve the most lexically-similar active items of one kind, ranked by - * idf-weighted token overlap computed entirely in SQL. Common tokens (high - * document frequency within this workspace + kind) are down-weighted so a - * shared rare term outranks several shared filler characters. - * - * Token rows only exist for active items (see schema invariant), so no - * status filter is needed here. - */ -const findDedupCandidatesEffect: MemoryRepository["findDedupCandidatesEffect"] = - (workspaceId, kind, tokens, limit) => - Effect.gen(function* () { - const db = yield* DbClient - if (tokens.length === 0 || limit <= 0) return [] - - const tokenList = sql.join( - tokens.map((token) => sql`${token}`), - sql`, `, - ) - - const scored = yield* Effect.promise(() => - db.execute<{ itemId: string }>(sql` - SELECT t.item_id AS "itemId", SUM(t.frequency::float8 / df.df) AS score - FROM fluid_memory_tokens t - JOIN ( - SELECT token, COUNT(DISTINCT item_id)::float8 AS df - FROM fluid_memory_tokens - WHERE workspace_id = ${workspaceId}::uuid - AND kind = ${kind} - AND token IN (${tokenList}) - GROUP BY token - ) df ON df.token = t.token - WHERE t.workspace_id = ${workspaceId}::uuid - AND t.kind = ${kind} - AND t.token IN (${tokenList}) - GROUP BY t.item_id - ORDER BY score DESC - LIMIT ${limit} - `), - ) - - const orderedIds = getRawRows(scored).map((row) => row.itemId) - if (orderedIds.length === 0) return [] - - const items = yield* Effect.promise(() => - db - .select() - .from(fluidMemoryItems) - .where(inArray(fluidMemoryItems.id, orderedIds)), - ) - const byId = new Map(items.map((item) => [item.id, item] as const)) - return orderedIds.flatMap((id) => { - const item = byId.get(id) - return item ? [item] : [] - }) - }) - -/** - * Append-only write of coarse-capture clues. Never touches fluid_memory_items. - * Empty input is a no-op (returns []). Status is always `pending`. - */ -const insertObservationsEffect: MemoryRepository["insertObservationsEffect"] = ( - input, -) => - Effect.gen(function* () { - const db = yield* DbClient - if (input.observations.length === 0) return [] - - const documentIds = [...input.referencedDocumentIds] - return yield* Effect.promise(() => - db - .insert(fluidObservations) - .values( - input.observations.map((observation) => ({ - workspaceId: input.workspaceId, - sourceMessageId: input.sourceMessageId, - signal: observation.signal, - evidenceQuote: observation.evidenceQuote, - subjectHint: observation.subjectHint ?? null, - referencedDocumentIds: documentIds, - confidence: observation.confidence, - status: "pending", - })), - ) - .returning(), - ) - }) - -const countPendingObservationsEffect: MemoryRepository["countPendingObservationsEffect"] = - (workspaceId) => - Effect.gen(function* () { - const db = yield* DbClient - const rows = yield* Effect.promise(() => - db - .select({ value: count() }) - .from(fluidObservations) - .where( - and( - eq(fluidObservations.workspaceId, workspaceId), - eq(fluidObservations.status, "pending"), - ), - ), - ) - return Number(rows[0]?.value ?? 0) - }) - -/** - * Oldest-first pending batch for distill. Concurrency across distill runs for - * the same workspace is primarily gated by trigger cooldown + bucketed - * workflowRunId; consume below is conditional on status still being pending. - */ -const listPendingObservationsEffect: MemoryRepository["listPendingObservationsEffect"] = - (workspaceId, limit) => - Effect.gen(function* () { - const db = yield* DbClient - if (limit <= 0) return [] - return yield* Effect.promise(() => - db - .select() - .from(fluidObservations) - .where( - and( - eq(fluidObservations.workspaceId, workspaceId), - eq(fluidObservations.status, "pending"), - ), - ) - .orderBy(asc(fluidObservations.createdAt)) - .limit(limit), - ) - }) - -const applyDistillBatchEffect: MemoryRepository["applyDistillBatchEffect"] = ( - input, -) => - Effect.gen(function* () { - const db = yield* DbClient - return yield* Effect.promise(() => - db.transaction(async (tx) => { - const diffs = await writeOperations( - tx, - input.workspaceId, - input.sourceMessageId, - input.operations, - ) - let consumedCount = 0 - if (input.observationIds.length > 0) { - const consumed = await tx - .update(fluidObservations) - .set({ - status: "consumed", - consumedAt: sql`now()`, - }) - .where( - and( - eq(fluidObservations.workspaceId, input.workspaceId), - eq(fluidObservations.status, "pending"), - inArray(fluidObservations.id, [...input.observationIds]), - ), - ) - .returning({ id: fluidObservations.id }) - consumedCount = consumed.length - } - return { diffs, consumedCount } - }), - ) - }) - -const deleteExpiredConsumedObservationsEffect: MemoryRepository["deleteExpiredConsumedObservationsEffect"] = - (olderThan) => - Effect.gen(function* () { - const db = yield* DbClient - const deleted = yield* Effect.promise(() => - db - .delete(fluidObservations) - .where( - and( - eq(fluidObservations.status, "consumed"), - lt(fluidObservations.createdAt, olderThan), - ), - ) - .returning({ id: fluidObservations.id }), - ) - return deleted.length - }) - -const listActiveItemsEffect: MemoryRepository["listActiveItemsEffect"] = ( - workspaceId, -) => - Effect.gen(function* () { - const db = yield* DbClient - return yield* Effect.promise(() => - db - .select({ - id: fluidMemoryItems.id, - kind: fluidMemoryItems.kind, - createdAt: fluidMemoryItems.createdAt, - }) - .from(fluidMemoryItems) - .where( - and( - eq(fluidMemoryItems.workspaceId, workspaceId), - eq(fluidMemoryItems.status, "active"), - ), - ), - ) - }) - -const deactivateDecayedItemsEffect: MemoryRepository["deactivateDecayedItemsEffect"] = - (workspaceId, itemIds) => - Effect.gen(function* () { - if (itemIds.length === 0) return 0 - - const db = yield* DbClient - return yield* Effect.promise(() => - db.transaction(async (tx) => { - const updated = await tx - .update(fluidMemoryItems) - .set({ - status: "inactive", - deactivationReason: - "decayed" satisfies FluidMemoryDeactivationReason, - updatedAt: sql`now()`, - }) - .where( - and( - eq(fluidMemoryItems.workspaceId, workspaceId), - eq(fluidMemoryItems.status, "active"), - inArray(fluidMemoryItems.id, [...itemIds]), - ), - ) - .returning({ id: fluidMemoryItems.id }) - - if (updated.length > 0) { - await tx.delete(fluidMemoryTokens).where( - inArray( - fluidMemoryTokens.itemId, - updated.map((item) => item.id), - ), - ) - } - return updated.length - }), - ) - }) - -export const memoryRepository: MemoryRepository = { - findDedupCandidatesEffect, - insertObservationsEffect, - countPendingObservationsEffect, - listPendingObservationsEffect, - applyDistillBatchEffect, - deleteExpiredConsumedObservationsEffect, - listActiveItemsEffect, - deactivateDecayedItemsEffect, -} - -async function writeOperations( - tx: TxClient, - workspaceId: string, - sourceMessageId: string | null, - operations: readonly ResolvedMemoryOperation[], -): Promise { - const diffOperations: MemoryDiffOperation[] = [] - - for (const operation of operations) { - switch (operation.op) { - case "create": { - const [inserted] = await tx - .insert(fluidMemoryItems) - .values({ - workspaceId, - kind: operation.kind, - payload: operation.payload, - abstractL0: operation.abstractL0, - overviewL1: operation.overviewL1, - sourceMessageId, - confidence: operation.confidence, - status: "active", - }) - .returning() - if (inserted?.id) { - const tokenRows = tokenRowsFor( - workspaceId, - inserted.id, - operation.kind, - operation.payload, - ) - if (tokenRows.length > 0) { - await tx.insert(fluidMemoryTokens).values(tokenRows) - } - } - diffOperations.push(toDiffOperation(operation, inserted?.id)) - break - } - case "merge": { - const [updated] = await tx - .update(fluidMemoryItems) - .set({ - payload: operation.payload, - abstractL0: operation.abstractL0, - overviewL1: operation.overviewL1, - confidence: operation.confidence, - sourceMessageId, - version: sql`${fluidMemoryItems.version} + 1`, - updatedAt: sql`now()`, - }) - .where( - and( - eq(fluidMemoryItems.id, operation.targetItemId), - eq(fluidMemoryItems.status, "active"), - ), - ) - .returning({ id: fluidMemoryItems.id }) - if (updated) { - await tx - .delete(fluidMemoryTokens) - .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) - const tokenRows = tokenRowsFor( - workspaceId, - operation.targetItemId, - operation.kind, - operation.payload, - ) - if (tokenRows.length > 0) { - await tx.insert(fluidMemoryTokens).values(tokenRows) - } - } - diffOperations.push( - updated - ? toDiffOperation(operation) - : { - op: "skip", - kind: operation.kind, - summary: operation.summary, - reason: "merge target no longer active", - }, - ) - break - } - case "deprecate": { - const [updated] = await tx - .update(fluidMemoryItems) - .set({ - status: "inactive", - deactivationReason: "contradicted" satisfies FluidMemoryDeactivationReason, - updatedAt: sql`now()`, - }) - .where( - and( - eq(fluidMemoryItems.id, operation.targetItemId), - eq(fluidMemoryItems.status, "active"), - ), - ) - .returning({ id: fluidMemoryItems.id }) - if (updated) { - await tx - .delete(fluidMemoryTokens) - .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) - } - diffOperations.push( - updated - ? toDiffOperation(operation) - : { - op: "skip", - kind: operation.kind, - summary: operation.summary, - reason: "deprecate target no longer active", - }, - ) - break - } - case "skip": - diffOperations.push(toDiffOperation(operation)) - break - } - } - - if (diffOperations.length > 0) { - await tx.insert(memoryDiffs).values({ - workspaceId, - sourceMessageId, - operations: [...diffOperations], - }) - } - - return diffOperations -} - -function tokenRowsFor( - workspaceId: string, - itemId: string, - kind: FluidMemoryKind, - payload: FluidMemoryPayload, -): NewFluidMemoryToken[] { - return buildMemoryItemTokens(kind, payload).map((token) => ({ - workspaceId, - itemId, - kind, - token: token.token, - frequency: token.frequency, - })) -} - -function getRawRows(value: RawRowsResult): readonly Row[] { - if (Array.isArray(value)) return value - return (value as { readonly rows: readonly Row[] }).rows -} diff --git a/src/domains/memory/resolve-operations.test.ts b/src/domains/memory/resolve-operations.test.ts deleted file mode 100644 index 03484b9..0000000 --- a/src/domains/memory/resolve-operations.test.ts +++ /dev/null @@ -1,378 +0,0 @@ -import { describe, expect, it } from "vitest" - -import type { MemoryOperations } from "./resolve-operations" -import { - resolveMemoryOperations, - toDiffOperation, -} from "./resolve-operations" - -const existingItems = [ - { - id: "item-1", - kind: "indicator_pref", - status: "active", - payload: { - name: "毛利率", - aliases: ["gross margin"], - definition: "毛利占营收的比例", - polarity: "higher_better", - importance: "core", - }, - }, - { id: "item-2", kind: "stance", status: "active" }, - { id: "item-3", kind: "stance", status: "inactive" }, - { - id: "item-4", - kind: "entity_of_interest", - status: "active", - payload: { - name: "英伟达", - ticker: "NVDA", - aliases: ["NVIDIA"], - knowhereDocumentIds: ["doc-earlier"], - reason: "用户持续跟踪", - }, - }, -] as const - -function makeOperations( - overrides: Partial = {}, -): MemoryOperations { - return { - indicatorPrefs: [], - stances: [], - decisionRules: [], - entities: [], - ...overrides, - } -} - -function makeIndicatorEntry(decision: { - op: "create" | "skip" | "merge" | "deprecate" - targetItemId?: string - reason?: string -}) { - return { - name: "毛利率", - aliases: ["gross margin"], - definition: "毛利占营收的比例", - polarity: "higher_better" as const, - importance: "core" as const, - abstractL0: "用户看重毛利率", - overviewL1: "用户在分析公司时首先看毛利率。", - confidence: 0.9, - decision, - } -} - -describe("resolveMemoryOperations", () => { - it("passes create through and ignores a stray targetItemId", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - makeIndicatorEntry({ op: "create", targetItemId: "item-1" }), - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved).toHaveLength(1) - expect(resolved[0]).toMatchObject({ - op: "create", - kind: "indicator_pref", - payload: { - name: "毛利率", - aliases: ["gross margin"], - polarity: "higher_better", - }, - }) - expect(resolved[0]).not.toHaveProperty("targetItemId") - }) - - it("merges into an existing active item of the same kind", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - makeIndicatorEntry({ op: "merge", targetItemId: "item-1" }), - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "merge", - kind: "indicator_pref", - targetItemId: "item-1", - }) - }) - - it("downgrades merge to skip when the target is missing", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - makeIndicatorEntry({ op: "merge", targetItemId: "item-999" }), - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "skip", - kind: "indicator_pref", - }) - expect(resolved[0]).not.toHaveProperty("targetItemId") - }) - - it("downgrades merge to skip on kind mismatch", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - makeIndicatorEntry({ op: "merge", targetItemId: "item-2" }), - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]?.op).toBe("skip") - }) - - it("downgrades merge to skip when the target is already inactive", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - stances: [ - { - statement: "长期持有，忽略短期波动", - scope: "投资", - rationale: "用户做长期投资", - abstractL0: "长期投资立场", - overviewL1: "用户强调长期持有。", - confidence: 1, - decision: { op: "merge", targetItemId: "item-3" }, - }, - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]?.op).toBe("skip") - }) - - it("keeps deprecate for an active target", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - stances: [ - { - statement: "美联储短期观点不重要", - scope: "宏观", - rationale: "长期投资", - abstractL0: "不看美联储短期观点", - overviewL1: "用户认为美联储短期观点权重低。", - confidence: 1, - decision: { - op: "deprecate", - targetItemId: "item-2", - reason: "用户改口", - }, - }, - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "deprecate", - targetItemId: "item-2", - reason: "用户改口", - }) - }) - - it("filters entity document ids to the turn's referenced documents", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - entities: [ - { - name: "英伟达", - ticker: "NVDA", - aliases: ["NVIDIA"], - knowhereDocumentIds: ["doc-real", "doc-hallucinated"], - reason: "用户持续跟踪", - abstractL0: "用户关注英伟达", - overviewL1: "用户多次询问英伟达财报。", - confidence: 0.8, - decision: { op: "create" }, - }, - ], - }), - existingItems, - referencedDocumentIds: ["doc-real"], - }) - - expect(resolved[0]).toMatchObject({ - op: "create", - kind: "entity_of_interest", - payload: { - name: "英伟达", - knowhereDocumentIds: ["doc-real"], - }, - }) - }) - - it("unions stored document ids when merging an entity", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - entities: [ - { - name: "英伟达", - ticker: "NVDA", - aliases: ["NVDA Corp"], - knowhereDocumentIds: ["doc-this-turn"], - reason: "用户持续跟踪", - abstractL0: "用户关注英伟达", - overviewL1: "用户多次询问英伟达财报。", - confidence: 0.9, - decision: { op: "merge", targetItemId: "item-4" }, - }, - ], - }), - existingItems, - referencedDocumentIds: ["doc-this-turn"], - }) - - expect(resolved[0]).toMatchObject({ - op: "merge", - targetItemId: "item-4", - payload: { - aliases: ["NVIDIA", "NVDA Corp"], - knowhereDocumentIds: ["doc-earlier", "doc-this-turn"], - }, - }) - }) - - it("unions aliases when merging an indicator preference", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - { - name: "毛利率", - aliases: ["同比毛利率"], - definition: "毛利占营收的比例，也看同比", - polarity: "higher_better" as const, - importance: "core" as const, - abstractL0: "毛利率也看同比", - overviewL1: "用户补充了同比视角。", - confidence: 1, - decision: { op: "merge", targetItemId: "item-1" }, - }, - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "merge", - targetItemId: "item-1", - payload: { - aliases: ["gross margin", "同比毛利率"], - }, - }) - }) - - it("skips create when the payload has no searchable tokens", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - { - name: "!!!", - aliases: [], - definition: "???", - polarity: "context" as const, - importance: "secondary" as const, - abstractL0: "无效符号", - overviewL1: "无法检索。", - confidence: 0.1, - decision: { op: "create" }, - }, - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "skip", - kind: "indicator_pref", - reason: "payload has no searchable tokens", - }) - }) - - it("skips merge when the merged payload has no searchable tokens", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - stances: [ - { - statement: "!!!", - scope: "???", - rationale: "...", - abstractL0: "无效符号", - overviewL1: "无法检索。", - confidence: 0.1, - decision: { op: "merge", targetItemId: "item-2" }, - }, - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "skip", - kind: "stance", - reason: "merged payload has no searchable tokens", - }) - }) -}) - -describe("toDiffOperation", () => { - it("records create with the inserted item id", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [makeIndicatorEntry({ op: "create" })], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(toDiffOperation(resolved[0]!, "new-id")).toEqual({ - op: "create", - kind: "indicator_pref", - summary: "用户看重毛利率", - itemId: "new-id", - }) - }) - - it("records merge/deprecate with their target item id", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - makeIndicatorEntry({ op: "merge", targetItemId: "item-1" }), - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(toDiffOperation(resolved[0]!)).toEqual({ - op: "merge", - kind: "indicator_pref", - summary: "用户看重毛利率", - itemId: "item-1", - }) - }) -}) diff --git a/src/domains/memory/resolve-operations.ts b/src/domains/memory/resolve-operations.ts deleted file mode 100644 index 833b955..0000000 --- a/src/domains/memory/resolve-operations.ts +++ /dev/null @@ -1,314 +0,0 @@ -import type { MemoryOperations } from "./distill-types" -import { buildMemoryItemTokens } from "./search-index" -import { - parseFluidMemoryPayload, - type FluidMemoryKind, - type FluidMemoryPayload, - type MemoryDiffOperation, -} from "./types" - -export type { MemoryOperations } from "./distill-types" - -/** - * Pure normalization from raw LLM operations to repository-ready - * operations. The LLM output already passed zod validation; this layer - * enforces the invariants the schema cannot express: - * - merge/deprecate must target an existing active item of the same kind - * (otherwise downgraded to skip — conservative, never fabricates) - * - entity knowhereDocumentIds are intersected with the document ids - * allowed for the batch (the model cannot invent provenance) - * - create ignores any targetItemId the model may have emitted - * - create/merge payloads must yield at least one lexical token, otherwise - * the item could never be retrieved for later dedup - * - merge unions aliases (and entity document ids) with the target so - * prior search terms / provenance are not wiped by a partial rewrite - * - * Used by distill (not by per-turn capture). Capture only writes observations. - */ - -export type ResolvedMemoryOperation = - | { - readonly op: "create" - readonly kind: FluidMemoryKind - readonly payload: FluidMemoryPayload - readonly abstractL0: string - readonly overviewL1: string - readonly confidence: number - readonly summary: string - readonly reason?: string - } - | { - readonly op: "skip" - readonly kind: FluidMemoryKind - readonly summary: string - readonly reason?: string - } - | { - readonly op: "merge" - readonly kind: FluidMemoryKind - readonly targetItemId: string - readonly payload: FluidMemoryPayload - readonly abstractL0: string - readonly overviewL1: string - readonly confidence: number - readonly summary: string - readonly reason?: string - } - | { - readonly op: "deprecate" - readonly kind: FluidMemoryKind - readonly targetItemId: string - readonly summary: string - readonly reason?: string - } - -export type ExistingMemoryItemRef = { - readonly id: string - readonly kind: string - readonly status: string - readonly payload?: unknown -} - -const kindToArrayKey = { - indicator_pref: "indicatorPrefs", - stance: "stances", - decision_rule: "decisionRules", - entity_of_interest: "entities", -} as const - -export function resolveMemoryOperations(input: { - readonly operations: MemoryOperations - readonly existingItems: readonly ExistingMemoryItemRef[] - readonly referencedDocumentIds: readonly string[] -}): ResolvedMemoryOperation[] { - const activeById = new Map( - input.existingItems - .filter((item) => item.status === "active") - .map((item) => [item.id, item] as const), - ) - const allowedDocumentIds = new Set(input.referencedDocumentIds) - - const resolved: ResolvedMemoryOperation[] = [] - - for (const kind of Object.keys(kindToArrayKey) as FluidMemoryKind[]) { - const entries = input.operations[kindToArrayKey[kind]] - - for (const entry of entries) { - const summary = entry.abstractL0 - const reason = entry.decision.reason - - if (entry.decision.op === "skip") { - resolved.push({ op: "skip", kind, summary, ...(reason ? { reason } : {}) }) - continue - } - - if (entry.decision.op === "merge" || entry.decision.op === "deprecate") { - const targetId = entry.decision.targetItemId - const target = targetId ? activeById.get(targetId) : undefined - if (!target || target.kind !== kind) { - resolved.push({ - op: "skip", - kind, - summary, - reason: `${entry.decision.op} target missing, inactive, or kind mismatch`, - }) - continue - } - if (entry.decision.op === "deprecate") { - resolved.push({ - op: "deprecate", - kind, - targetItemId: target.id, - summary, - ...(reason ? { reason } : {}), - }) - continue - } - const mergePayload = toPayload( - kind, - entry as Record, - allowedDocumentIds, - ) - if (!mergePayload) { - resolved.push({ - op: "skip", - kind, - summary, - reason: "merged payload failed validation", - }) - continue - } - const preserved = preserveFieldsOnMerge( - kind, - mergePayload, - target.payload, - ) - if (!isIndexable(kind, preserved)) { - resolved.push({ - op: "skip", - kind, - summary, - reason: "merged payload has no searchable tokens", - }) - continue - } - resolved.push({ - op: "merge", - kind, - targetItemId: target.id, - payload: preserved, - abstractL0: entry.abstractL0, - overviewL1: entry.overviewL1, - confidence: entry.confidence, - summary, - ...(reason ? { reason } : {}), - }) - continue - } - - const createPayload = toPayload( - kind, - entry as Record, - allowedDocumentIds, - ) - if (!createPayload) { - resolved.push({ - op: "skip", - kind, - summary, - reason: "payload failed validation", - }) - continue - } - if (!isIndexable(kind, createPayload)) { - resolved.push({ - op: "skip", - kind, - summary, - reason: "payload has no searchable tokens", - }) - continue - } - resolved.push({ - op: "create", - kind, - payload: createPayload, - abstractL0: entry.abstractL0, - overviewL1: entry.overviewL1, - confidence: entry.confidence, - summary, - ...(reason ? { reason } : {}), - }) - } - } - - return resolved -} - -function toPayload( - kind: FluidMemoryKind, - entry: Record, - allowedDocumentIds: ReadonlySet, -): FluidMemoryPayload | null { - const candidate: Record = { - name: entry.name, - aliases: entry.aliases, - definition: entry.definition, - polarity: entry.polarity, - importance: entry.importance, - formulaHint: entry.formulaHint, - statement: entry.statement, - scope: entry.scope, - rationale: entry.rationale, - when: entry.when, - then: entry.then, - priority: entry.priority, - ticker: entry.ticker, - reason: entry.reason, - knowhereDocumentIds: Array.isArray(entry.knowhereDocumentIds) - ? entry.knowhereDocumentIds.filter( - (id): id is string => - typeof id === "string" && allowedDocumentIds.has(id), - ) - : [], - } - return parseFluidMemoryPayload(kind, candidate) -} - -/** Reject payloads that could never be found again by the token index. */ -function isIndexable(kind: FluidMemoryKind, payload: FluidMemoryPayload): boolean { - return buildMemoryItemTokens(kind, payload).length > 0 -} - -/** - * Merge replaces the stored payload, but the model only sees this batch. - * Union aliases (and entity document ids) with the target so earlier search - * terms / provenance survive a partial rewrite. - */ -function preserveFieldsOnMerge( - kind: FluidMemoryKind, - mergedPayload: FluidMemoryPayload, - existingPayload: unknown, -): FluidMemoryPayload { - const existing = parseFluidMemoryPayload(kind, existingPayload) - if (!existing) return mergedPayload - - if ( - kind === "indicator_pref" && - "aliases" in mergedPayload && - "aliases" in existing - ) { - return { - ...mergedPayload, - aliases: unionStrings(existing.aliases, mergedPayload.aliases), - } - } - - if ( - kind === "entity_of_interest" && - "aliases" in mergedPayload && - "aliases" in existing && - "knowhereDocumentIds" in mergedPayload && - "knowhereDocumentIds" in existing - ) { - return { - ...mergedPayload, - aliases: unionStrings(existing.aliases, mergedPayload.aliases), - knowhereDocumentIds: unionStrings( - existing.knowhereDocumentIds, - mergedPayload.knowhereDocumentIds, - ), - } - } - - return mergedPayload -} - -function unionStrings( - left: readonly string[], - right: readonly string[], -): string[] { - return [...new Set([...left, ...right])] -} - -/** Diff-audit view of a resolved operation (itemId filled after write). */ -export function toDiffOperation( - operation: ResolvedMemoryOperation, - itemId?: string, -): MemoryDiffOperation { - const base = { - kind: operation.kind, - summary: operation.summary, - ...(operation.reason ? { reason: operation.reason } : {}), - } - switch (operation.op) { - case "create": - return { op: "create", ...base, ...(itemId ? { itemId } : {}) } - case "merge": - return { op: "merge", ...base, itemId: operation.targetItemId } - case "deprecate": - return { op: "deprecate", ...base, itemId: operation.targetItemId } - case "skip": - return { op: "skip", ...base } - } -} diff --git a/src/domains/memory/search-index.test.ts b/src/domains/memory/search-index.test.ts deleted file mode 100644 index cbf5d0b..0000000 --- a/src/domains/memory/search-index.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { - buildMemoryItemTokens, - buildMemorySearchText, - tokenizeMemoryText, -} from "./search-index" - -describe("tokenizeMemoryText", () => { - it("splits CJK into single characters and latin into words", () => { - expect(tokenizeMemoryText("毛利率 PE gross_margin")).toEqual([ - { token: "毛", frequency: 1 }, - { token: "利", frequency: 1 }, - { token: "率", frequency: 1 }, - { token: "pe", frequency: 1 }, - { token: "gross_margin", frequency: 1 }, - ]) - }) - - it("counts repeated tokens", () => { - expect(tokenizeMemoryText("PE pe Pe")).toEqual([ - { token: "pe", frequency: 3 }, - ]) - }) - - it("returns empty for whitespace-only input", () => { - expect(tokenizeMemoryText(" ")).toEqual([]) - }) -}) - -describe("buildMemorySearchText", () => { - it("indexes indicator name, aliases, and definition", () => { - expect( - buildMemorySearchText("indicator_pref", { - name: "毛利率", - aliases: ["gross margin"], - definition: "毛利除以营收", - polarity: "higher_better", - importance: "core", - }), - ).toBe("毛利率 gross margin 毛利除以营收") - }) - - it("indexes entity name, aliases, and ticker without reason", () => { - expect( - buildMemorySearchText("entity_of_interest", { - name: "英伟达", - aliases: ["NVIDIA"], - ticker: "NVDA", - knowhereDocumentIds: ["doc-1"], - reason: "一直在跟踪", - }), - ).toBe("英伟达 NVIDIA NVDA") - }) - - it("indexes stance statement and scope", () => { - expect( - buildMemorySearchText("stance", { - statement: "做长期投资", - scope: "宏观短期观点", - rationale: "美联储短期说法不重要", - }), - ).toBe("做长期投资 宏观短期观点") - }) - - it("indexes decision rule when and then", () => { - expect( - buildMemorySearchText("decision_rule", { - when: "毛利率连续两季下滑", - then: "减仓观望", - priority: "high", - rationale: "用户明确说过", - }), - ).toBe("毛利率连续两季下滑 减仓观望") - }) -}) - -describe("buildMemoryItemTokens", () => { - it("tokenizes the search text of an item", () => { - const tokens = buildMemoryItemTokens("indicator_pref", { - name: "PE", - aliases: [], - definition: "市盈率", - polarity: "context", - importance: "secondary", - }) - expect(tokens.map((token) => token.token)).toEqual([ - "pe", - "市", - "盈", - "率", - ]) - }) -}) diff --git a/src/domains/memory/search-index.ts b/src/domains/memory/search-index.ts deleted file mode 100644 index 84a53e9..0000000 --- a/src/domains/memory/search-index.ts +++ /dev/null @@ -1,84 +0,0 @@ -import type { - DecisionRulePayload, - EntityOfInterestPayload, - FluidMemoryKind, - FluidMemoryPayload, - IndicatorPreferencePayload, - StancePayload, -} from "./types" - -/** - * Lexical search-index helpers for fluid memory dedup retrieval. - * - * Pure and dependency-free so both the write path (indexing an item) and the - * read path (turning a turn into a query) share one tokenizer. Tokenization - * mirrors Knowhere map-nav: lowercase, then emit single CJK characters and - * `[a-z0-9_]+` runs. This handles Chinese (no whitespace segmentation) and - * Latin/alphanumeric terms without any Postgres extension. - */ - -export type MemoryToken = { - readonly token: string - readonly frequency: number -} - -// Single CJK char OR a run of latin letters / digits / underscore. -const TOKEN_PATTERN = - /[a-z0-9_]+|[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/g - -/** - * Build the text that represents an item for lexical matching. Only the - * fields a user would phrase a query against are included (names, aliases, - * short definitions), not provenance or bookkeeping fields. - */ -export function buildMemorySearchText( - kind: FluidMemoryKind, - payload: FluidMemoryPayload, -): string { - return collectSearchParts(kind, payload) - .filter((part) => part.length > 0) - .join(" ") -} - -/** Tokenize free text into deduped tokens with occurrence counts. */ -export function tokenizeMemoryText(text: string): MemoryToken[] { - const counts = new Map() - const matches = text.toLowerCase().match(TOKEN_PATTERN) - if (!matches) return [] - for (const token of matches) { - counts.set(token, (counts.get(token) ?? 0) + 1) - } - return [...counts].map(([token, frequency]) => ({ token, frequency })) -} - -/** Tokens that index one memory item (search text of its payload). */ -export function buildMemoryItemTokens( - kind: FluidMemoryKind, - payload: FluidMemoryPayload, -): MemoryToken[] { - return tokenizeMemoryText(buildMemorySearchText(kind, payload)) -} - -function collectSearchParts( - kind: FluidMemoryKind, - payload: FluidMemoryPayload, -): readonly string[] { - switch (kind) { - case "indicator_pref": { - const p = payload as IndicatorPreferencePayload - return [p.name, ...p.aliases, p.definition] - } - case "stance": { - const p = payload as StancePayload - return [p.statement, p.scope] - } - case "decision_rule": { - const p = payload as DecisionRulePayload - return [p.when, p.then] - } - case "entity_of_interest": { - const p = payload as EntityOfInterestPayload - return [p.name, ...p.aliases, ...(p.ticker ? [p.ticker] : [])] - } - } -} diff --git a/src/domains/memory/service.ts b/src/domains/memory/service.ts deleted file mode 100644 index f711f98..0000000 --- a/src/domains/memory/service.ts +++ /dev/null @@ -1,146 +0,0 @@ -import "server-only" - -import { databaseRuntime } from "@/domains/workspace/database-runtime" -import { selectDecayCandidates, type DecayCandidate } from "./decay-candidates" -import { - memoryRepository, - type ApplyDistillBatchInput, - type InsertObservationsInput, -} from "./repository" -import type { FluidMemoryKind, MemoryDiffOperation } from "./types" -import { retrievalActivationService } from "@/domains/retrieval-activation/service" -import type { - FluidMemoryItem, - FluidObservation, -} from "@/infrastructure/db/schema" - -type MemoryService = { - readonly findDedupCandidates: ( - workspaceId: string, - kind: FluidMemoryKind, - tokens: readonly string[], - limit: number, - ) => Promise - readonly insertObservations: ( - input: InsertObservationsInput, - ) => Promise - readonly countPendingObservations: (workspaceId: string) => Promise - readonly listPendingObservations: ( - workspaceId: string, - limit: number, - ) => Promise - readonly applyDistillBatch: (input: ApplyDistillBatchInput) => Promise<{ - readonly diffs: readonly MemoryDiffOperation[] - readonly consumedCount: number - }> - readonly deleteExpiredConsumedObservations: ( - olderThan: Date, - ) => Promise - /** - * Active items whose activation-decay score is below `scoreThreshold`. - * Read-only — does not change any item's status. The caller decides the - * threshold and what to do with the result (see - * `deactivateDecayedItems` to actually move candidates to `inactive`). - */ - readonly listDecayCandidates: ( - workspaceId: string, - options: { readonly now: Date; readonly scoreThreshold: number }, - ) => Promise - readonly deactivateDecayedItems: ( - workspaceId: string, - itemIds: readonly string[], - ) => Promise -} - -const findDedupCandidates: MemoryService["findDedupCandidates"] = ( - workspaceId, - kind, - tokens, - limit, -) => - databaseRuntime.runPromise( - memoryRepository.findDedupCandidatesEffect( - workspaceId, - kind, - tokens, - limit, - ), - ) - -const insertObservations: MemoryService["insertObservations"] = (input) => - databaseRuntime.runPromise(memoryRepository.insertObservationsEffect(input)) - -const countPendingObservations: MemoryService["countPendingObservations"] = ( - workspaceId, -) => - databaseRuntime.runPromise( - memoryRepository.countPendingObservationsEffect(workspaceId), - ) - -const listPendingObservations: MemoryService["listPendingObservations"] = ( - workspaceId, - limit, -) => - databaseRuntime.runPromise( - memoryRepository.listPendingObservationsEffect(workspaceId, limit), - ) - -const applyDistillBatch: MemoryService["applyDistillBatch"] = (input) => - databaseRuntime.runPromise(memoryRepository.applyDistillBatchEffect(input)) - -const deleteExpiredConsumedObservations: MemoryService["deleteExpiredConsumedObservations"] = - (olderThan) => - databaseRuntime.runPromise( - memoryRepository.deleteExpiredConsumedObservationsEffect(olderThan), - ) - -const listDecayCandidates: MemoryService["listDecayCandidates"] = async ( - workspaceId, - options, -) => { - const items = await databaseRuntime.runPromise( - memoryRepository.listActiveItemsEffect(workspaceId), - ) - if (items.length === 0) return [] - - const activations = await retrievalActivationService.getActivations( - workspaceId, - "fluid_memory", - items.map((item) => item.id), - ) - const activationsById = new Map( - activations.map((activation) => [ - activation.unitRef, - { - activationCount: activation.activationCount, - lastActivatedAt: activation.lastActivatedAt, - }, - ]), - ) - - return selectDecayCandidates({ - items, - activationsById, - now: options.now, - scoreThreshold: options.scoreThreshold, - }) -} - -const deactivateDecayedItems: MemoryService["deactivateDecayedItems"] = ( - workspaceId, - itemIds, -) => - databaseRuntime.runPromise( - memoryRepository.deactivateDecayedItemsEffect(workspaceId, itemIds), - ) - -export const memoryService: MemoryService = { - findDedupCandidates, - insertObservations, - countPendingObservations, - listPendingObservations, - applyDistillBatch, - deleteExpiredConsumedObservations, - listDecayCandidates, - deactivateDecayedItems, -} diff --git a/src/domains/memory/types.ts b/src/domains/memory/types.ts deleted file mode 100644 index b9975cb..0000000 --- a/src/domains/memory/types.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { z } from "zod" - -/** - * Fluid memory type contract. - * - * Four typed payload kinds, extracted from conversation turns. The DB - * stores `payload` as jsonb; these schemas are the validation boundary on - * both write (LLM output) and read (repository decode) paths. - */ - -export const fluidMemoryKinds = [ - "indicator_pref", - "stance", - "decision_rule", - "entity_of_interest", -] as const - -export type FluidMemoryKind = (typeof fluidMemoryKinds)[number] - -export const indicatorPreferencePayloadSchema = z.object({ - name: z.string().min(1), - aliases: z.array(z.string()).default([]), - definition: z.string().min(1), - polarity: z.enum(["higher_better", "lower_better", "context"]), - importance: z.enum(["core", "secondary"]), - formulaHint: z.preprocess( - (value) => (value === null ? undefined : value), - z.string().optional(), - ), -}) - -export const stancePayloadSchema = z.object({ - statement: z.string().min(1), - scope: z.string().min(1), - rationale: z.string().min(1), -}) - -export const decisionRulePayloadSchema = z.object({ - when: z.string().min(1), - then: z.string().min(1), - priority: z.enum(["high", "medium", "low"]), - rationale: z.string().min(1), -}) - -export const entityOfInterestPayloadSchema = z.object({ - name: z.string().min(1), - ticker: z.preprocess( - (value) => (value === null ? undefined : value), - z.string().optional(), - ), - aliases: z.array(z.string()).default([]), - knowhereDocumentIds: z.array(z.string()).default([]), - reason: z.string().min(1), -}) - -export type IndicatorPreferencePayload = z.infer< - typeof indicatorPreferencePayloadSchema -> -export type StancePayload = z.infer -export type DecisionRulePayload = z.infer -export type EntityOfInterestPayload = z.infer< - typeof entityOfInterestPayloadSchema -> - -export type FluidMemoryPayload = - | IndicatorPreferencePayload - | StancePayload - | DecisionRulePayload - | EntityOfInterestPayload - -const payloadSchemas: Record> = { - indicator_pref: indicatorPreferencePayloadSchema, - stance: stancePayloadSchema, - decision_rule: decisionRulePayloadSchema, - entity_of_interest: entityOfInterestPayloadSchema, -} - -export function isFluidMemoryKind(value: unknown): value is FluidMemoryKind { - return ( - typeof value === "string" && - (fluidMemoryKinds as readonly string[]).includes(value) - ) -} - -/** Decode a persisted jsonb payload; returns null when the row is malformed. */ -export function parseFluidMemoryPayload( - kind: FluidMemoryKind, - value: unknown, -): FluidMemoryPayload | null { - const result = payloadSchemas[kind].safeParse(value) - return result.success ? result.data : null -} - -/** - * Why an item left `active` for `inactive`. Orthogonal to `status`: `status` - * says whether the item is retrievable today, `deactivationReason` says - * which mechanism moved it out. - * - contradicted — distill decided a later turn reverses this item - * - decayed — the activation-decay job flagged it as unused past threshold - */ -export const fluidMemoryDeactivationReasons = ["contradicted", "decayed"] as const -export type FluidMemoryDeactivationReason = - (typeof fluidMemoryDeactivationReasons)[number] - -/** One decided operation over the memory set; persisted into memory_diffs. */ -export type MemoryDiffOperation = { - readonly op: "create" | "skip" | "merge" | "deprecate" - readonly kind: FluidMemoryKind - readonly itemId?: string - readonly summary: string - readonly reason?: string -} diff --git a/src/domains/retrieval-activation/decay-score.test.ts b/src/domains/retrieval-activation/decay-score.test.ts deleted file mode 100644 index db239eb..0000000 --- a/src/domains/retrieval-activation/decay-score.test.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { BASE_HALF_LIFE_DAYS, computeDecayScore } from "./decay-score" - -const NOW = new Date("2026-01-08T00:00:00Z") - -describe("computeDecayScore", () => { - it("scores a freshly anchored, never-activated unit as neutral 0.5", () => { - // freq = sigmoid(log1p(0)) = sigmoid(0) = 0.5; recency at age 0 = 1. - const score = computeDecayScore({ - activationCount: 0, - anchorAt: NOW, - now: NOW, - }) - expect(score).toBeCloseTo(0.5, 10) - }) - - it("halves the neutral score after one base half-life with no activations", () => { - const anchorAt = new Date( - NOW.getTime() - BASE_HALF_LIFE_DAYS * 24 * 60 * 60 * 1000, - ) - const score = computeDecayScore({ activationCount: 0, anchorAt, now: NOW }) - expect(score).toBeCloseTo(0.25, 10) - }) - - it("decays monotonically with age for a fixed activation count", () => { - const dayAgo = new Date(NOW.getTime() - 24 * 60 * 60 * 1000) - const weekAgo = new Date(NOW.getTime() - 7 * 24 * 60 * 60 * 1000) - - const scoreAtDay = computeDecayScore({ - activationCount: 2, - anchorAt: dayAgo, - now: NOW, - }) - const scoreAtWeek = computeDecayScore({ - activationCount: 2, - anchorAt: weekAgo, - now: NOW, - }) - - expect(scoreAtDay).toBeGreaterThan(scoreAtWeek) - }) - - it("scores a higher activation count above a lower one at the same age", () => { - const weekAgo = new Date(NOW.getTime() - 7 * 24 * 60 * 60 * 1000) - - const lowCount = computeDecayScore({ - activationCount: 1, - anchorAt: weekAgo, - now: NOW, - }) - const highCount = computeDecayScore({ - activationCount: 10, - anchorAt: weekAgo, - now: NOW, - }) - - expect(highCount).toBeGreaterThan(lowCount) - }) - - it("resets toward the frequency ceiling immediately after a fresh activation", () => { - // A unit with a long activation history but an activation just now - // should score close to its frequency ceiling, not its pre-reset decay. - const score = computeDecayScore({ - activationCount: 5, - anchorAt: NOW, - now: NOW, - }) - const frequencyCeiling = 1 / (1 + Math.exp(-Math.log1p(5))) - expect(score).toBeCloseTo(frequencyCeiling, 10) - }) - - it("stays within (0, 1) across a range of counts and ages", () => { - const activationCounts = [0, 1, 3, 10, 50] - const ageDaysList = [0, 1, 7, 30, 365] - - for (const activationCount of activationCounts) { - for (const ageDays of ageDaysList) { - const anchorAt = new Date( - NOW.getTime() - ageDays * 24 * 60 * 60 * 1000, - ) - const score = computeDecayScore({ activationCount, anchorAt, now: NOW }) - expect(score).toBeGreaterThan(0) - expect(score).toBeLessThan(1) - } - } - }) - - it("clamps negative age (anchor in the future) to zero elapsed time", () => { - const future = new Date(NOW.getTime() + 24 * 60 * 60 * 1000) - const score = computeDecayScore({ - activationCount: 0, - anchorAt: future, - now: NOW, - }) - expect(score).toBeCloseTo(0.5, 10) - }) -}) diff --git a/src/domains/retrieval-activation/decay-score.ts b/src/domains/retrieval-activation/decay-score.ts deleted file mode 100644 index 0e2ca1b..0000000 --- a/src/domains/retrieval-activation/decay-score.ts +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Time-decay importance score for a retrievable unit (fluid memory item or - * crystal chunk). Pure function, computed at read time — never persisted — - * matching OpenViking's approach of not storing a score that would need - * migration whenever the formula changes. - * - * Structure copied from OpenViking's `hotness_score` - * (`sigmoid(log1p(activationCount)) × exp(-ln2/halfLife × ageDays)`), see - * `.repos/OpenViking/openviking/retrieve/memory_lifecycle.py`. - * - * `BASE_HALF_LIFE_DAYS` is deliberately 2x OpenViking's own default (7 - * days) — a longer grace period before an unused unit's importance - * meaningfully drops, confirmed against simulated day-counts (see the - * `记忆衰减聚类收尾方案` plan for the numbers this was checked against). - * - * One deviation from OpenViking: the recency half-life grows with - * `activationCount` instead of staying fixed, borrowing MemoryBank's - * (arXiv:2305.10250) intuition that repeated recall makes a memory more - * resistant to forgetting (there, strength `S` is incremented by 1 on every - * recall and used directly as the decay time constant). Here: - * - * effectiveHalfLifeDays = BASE_HALF_LIFE_DAYS * (1 + activationCount) - * - * This does not double-count activationCount with the frequency term: the - * frequency term sets the score's baseline ceiling for a given activation - * count, while the half-life growth slows how fast that ceiling erodes as - * time passes without a new activation. - * - * "Reset then decay" (an activation makes the unit feel fresh again, then - * importance decays again from there) is achieved by the caller advancing - * `anchorAt` to the activation time on every write — this function only - * computes the curve from whatever anchor it is given. - */ - -const MS_PER_DAY = 24 * 60 * 60 * 1000 - -/** 2x OpenViking's DEFAULT_HALF_LIFE_DAYS (7) — see rationale above. */ -export const BASE_HALF_LIFE_DAYS = 14 - -export type DecayScoreInput = { - /** Total times this unit has been cited into an answer. */ - readonly activationCount: number - /** Last activation time, or the unit's creation time if never activated. */ - readonly anchorAt: Date - readonly now: Date -} - -/** Always in (0, 1). */ -export function computeDecayScore(input: DecayScoreInput): number { - const activationCount = Math.max(input.activationCount, 0) - const ageDays = Math.max( - (input.now.getTime() - input.anchorAt.getTime()) / MS_PER_DAY, - 0, - ) - - const frequency = sigmoid(Math.log1p(activationCount)) - const effectiveHalfLifeDays = BASE_HALF_LIFE_DAYS * (1 + activationCount) - const decayRate = Math.LN2 / effectiveHalfLifeDays - const recency = Math.exp(-decayRate * ageDays) - - return frequency * recency -} - -function sigmoid(x: number): number { - return 1 / (1 + Math.exp(-x)) -} diff --git a/src/domains/retrieval-activation/repository.test.ts b/src/domains/retrieval-activation/repository.test.ts deleted file mode 100644 index 267c81e..0000000 --- a/src/domains/retrieval-activation/repository.test.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest" -import { Effect, Layer } from "effect" - -import { retrievalActivationRepository } from "./repository" -import type { Db } from "@/infrastructure/db" - -type InsertValues = { - readonly workspaceId: string - readonly unitType: string - readonly unitRef: string - readonly activationCount: number -} - -afterEach(() => { - vi.restoreAllMocks() -}) - -async function runWithMockDb(insertValues: InsertValues[]) { - const insertBuilder = { - values: vi.fn((values: InsertValues[]) => { - insertValues.push(...values) - return insertBuilder - }), - onConflictDoUpdate: vi.fn(() => insertBuilder), - returning: vi.fn(async () => - insertValues.map((_, index) => ({ id: `activation_${index}` })), - ), - } - const dbMock = { insert: vi.fn(() => insertBuilder) } - const { DbClient } = await vi.importActual( - "@/infrastructure/db", - ) - const dbLayer = Layer.succeed(DbClient, dbMock as unknown as Db) - return { dbLayer, insertBuilder, dbMock } -} - -describe("retrievalActivationRepository.recordActivationsEffect", () => { - it("does nothing and never touches the db for an empty batch", async () => { - const insertValues: InsertValues[] = [] - const { dbLayer, dbMock } = await runWithMockDb(insertValues) - - const written = await Effect.runPromise( - retrievalActivationRepository - .recordActivationsEffect([]) - .pipe(Effect.provide(dbLayer)), - ) - - expect(written).toBe(0) - expect(dbMock.insert).not.toHaveBeenCalled() - }) - - it("collapses duplicate unit refs into a single upsert row", async () => { - const insertValues: InsertValues[] = [] - const { dbLayer, insertBuilder } = await runWithMockDb(insertValues) - - const written = await Effect.runPromise( - retrievalActivationRepository - .recordActivationsEffect([ - { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "doc:chunk_1" }, - { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "doc:chunk_1" }, - { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "doc:chunk_2" }, - ]) - .pipe(Effect.provide(dbLayer)), - ) - - expect(written).toBe(2) - expect(insertBuilder.values).toHaveBeenCalledOnce() - expect(insertValues).toHaveLength(2) - expect(insertValues.map((row) => row.unitRef).sort()).toEqual([ - "doc:chunk_1", - "doc:chunk_2", - ]) - }) - - it("keeps the same unit ref distinct across different unit types and workspaces", async () => { - const insertValues: InsertValues[] = [] - const { dbLayer } = await runWithMockDb(insertValues) - - await Effect.runPromise( - retrievalActivationRepository - .recordActivationsEffect([ - { workspaceId: "ws_1", unitType: "fluid_memory", unitRef: "item_1" }, - { workspaceId: "ws_1", unitType: "crystal_chunk", unitRef: "item_1" }, - { workspaceId: "ws_2", unitType: "fluid_memory", unitRef: "item_1" }, - ]) - .pipe(Effect.provide(dbLayer)), - ) - - expect(insertValues).toHaveLength(3) - }) -}) - -describe("retrievalActivationRepository.getActivationsEffect", () => { - it("returns [] without querying the db for an empty unit ref list", async () => { - const selectBuilder = { from: vi.fn(), where: vi.fn() } - const dbMock = { select: vi.fn(() => selectBuilder) } - const { DbClient } = await vi.importActual( - "@/infrastructure/db", - ) - const dbLayer = Layer.succeed(DbClient, dbMock as unknown as Db) - - const result = await Effect.runPromise( - retrievalActivationRepository - .getActivationsEffect("ws_1", "fluid_memory", []) - .pipe(Effect.provide(dbLayer)), - ) - - expect(result).toEqual([]) - expect(dbMock.select).not.toHaveBeenCalled() - }) - - it("returns matching activation rows", async () => { - const rows = [ - { - unitRef: "item_1", - activationCount: 3, - lastActivatedAt: new Date("2026-01-01T00:00:00Z"), - }, - ] - const selectBuilder = { - from: vi.fn(() => selectBuilder), - where: vi.fn(async () => rows), - } - const dbMock = { select: vi.fn(() => selectBuilder) } - const { DbClient } = await vi.importActual( - "@/infrastructure/db", - ) - const dbLayer = Layer.succeed(DbClient, dbMock as unknown as Db) - - const result = await Effect.runPromise( - retrievalActivationRepository - .getActivationsEffect("ws_1", "fluid_memory", ["item_1", "item_2"]) - .pipe(Effect.provide(dbLayer)), - ) - - expect(result).toEqual(rows) - }) -}) diff --git a/src/domains/retrieval-activation/repository.ts b/src/domains/retrieval-activation/repository.ts deleted file mode 100644 index 53732ac..0000000 --- a/src/domains/retrieval-activation/repository.ts +++ /dev/null @@ -1,119 +0,0 @@ -import "server-only" - -import { and, eq, inArray, sql } from "drizzle-orm" -import { Effect } from "effect" - -import type { RetrievalUnitType } from "./types" -import { DbClient } from "@/infrastructure/db" -import { retrievalActivations } from "@/infrastructure/db/schema" - -export type RecordActivationInput = { - readonly workspaceId: string - readonly unitType: RetrievalUnitType - readonly unitRef: string -} - -export type ActivationStats = { - readonly unitRef: string - readonly activationCount: number - readonly lastActivatedAt: Date | null -} - -type RetrievalActivationRepository = { - /** - * Upsert one activation (+1, lastActivatedAt = now) per distinct unit. - * Duplicate (workspaceId, unitType, unitRef) entries within `inputs` are - * collapsed to a single +1 — Postgres rejects a multi-row upsert that - * would touch the same conflict target twice in one statement, and - * "cited twice in one answer" should still only count as one activation - * event for this turn. - */ - readonly recordActivationsEffect: ( - inputs: readonly RecordActivationInput[], - ) => Effect.Effect - /** - * Existing ledger rows for a set of unit refs of one type. Units with no - * row (never activated) are simply absent from the result — the caller - * treats that as activationCount 0. - */ - readonly getActivationsEffect: ( - workspaceId: string, - unitType: RetrievalUnitType, - unitRefs: readonly string[], - ) => Effect.Effect -} - -const recordActivationsEffect: RetrievalActivationRepository["recordActivationsEffect"] = - (inputs) => - Effect.gen(function* () { - const deduped = dedupeInputs(inputs) - if (deduped.length === 0) return 0 - - const db = yield* DbClient - const written = yield* Effect.promise(() => - db - .insert(retrievalActivations) - .values( - deduped.map((input) => ({ - workspaceId: input.workspaceId, - unitType: input.unitType, - unitRef: input.unitRef, - activationCount: 1, - lastActivatedAt: sql`now()`, - })), - ) - .onConflictDoUpdate({ - target: [ - retrievalActivations.workspaceId, - retrievalActivations.unitType, - retrievalActivations.unitRef, - ], - set: { - activationCount: sql`${retrievalActivations.activationCount} + 1`, - lastActivatedAt: sql`now()`, - }, - }) - .returning({ id: retrievalActivations.id }), - ) - return written.length - }) - -const getActivationsEffect: RetrievalActivationRepository["getActivationsEffect"] = - (workspaceId, unitType, unitRefs) => - Effect.gen(function* () { - if (unitRefs.length === 0) return [] - - const db = yield* DbClient - const rows = yield* Effect.promise(() => - db - .select({ - unitRef: retrievalActivations.unitRef, - activationCount: retrievalActivations.activationCount, - lastActivatedAt: retrievalActivations.lastActivatedAt, - }) - .from(retrievalActivations) - .where( - and( - eq(retrievalActivations.workspaceId, workspaceId), - eq(retrievalActivations.unitType, unitType), - inArray(retrievalActivations.unitRef, [...unitRefs]), - ), - ), - ) - return rows - }) - -export const retrievalActivationRepository: RetrievalActivationRepository = { - recordActivationsEffect, - getActivationsEffect, -} - -function dedupeInputs( - inputs: readonly RecordActivationInput[], -): RecordActivationInput[] { - const byKey = new Map() - for (const input of inputs) { - byKey.set(`${input.workspaceId}\u0000${input.unitType}\u0000${input.unitRef}`, input) - } - return [...byKey.values()] -} diff --git a/src/domains/retrieval-activation/service.ts b/src/domains/retrieval-activation/service.ts deleted file mode 100644 index 990a6eb..0000000 --- a/src/domains/retrieval-activation/service.ts +++ /dev/null @@ -1,45 +0,0 @@ -import "server-only" - -import { - retrievalActivationRepository, - type ActivationStats, - type RecordActivationInput, -} from "./repository" -import type { RetrievalUnitType } from "./types" -import { databaseRuntime } from "@/domains/workspace/database-runtime" - -type RetrievalActivationService = { - readonly recordActivations: ( - inputs: readonly RecordActivationInput[], - ) => Promise - readonly getActivations: ( - workspaceId: string, - unitType: RetrievalUnitType, - unitRefs: readonly string[], - ) => Promise -} - -const recordActivations: RetrievalActivationService["recordActivations"] = ( - inputs, -) => - databaseRuntime.runPromise( - retrievalActivationRepository.recordActivationsEffect(inputs), - ) - -const getActivations: RetrievalActivationService["getActivations"] = ( - workspaceId, - unitType, - unitRefs, -) => - databaseRuntime.runPromise( - retrievalActivationRepository.getActivationsEffect( - workspaceId, - unitType, - unitRefs, - ), - ) - -export const retrievalActivationService: RetrievalActivationService = { - recordActivations, - getActivations, -} diff --git a/src/domains/retrieval-activation/types.ts b/src/domains/retrieval-activation/types.ts deleted file mode 100644 index 29e18db..0000000 --- a/src/domains/retrieval-activation/types.ts +++ /dev/null @@ -1,17 +0,0 @@ -/** - * A "unit" is anything retrieval can surface and an answer can actually - * cite. Today there are two kinds: - * - fluid_memory — a `fluid_memory_items` row, keyed by its id - * - crystal_chunk — a Knowhere chunk, which has no local row; keyed by - * `${documentId}:${chunkId}` (see `toChunkUnitRef`) - */ -export const retrievalUnitTypes = ["fluid_memory", "crystal_chunk"] as const -export type RetrievalUnitType = (typeof retrievalUnitTypes)[number] - -/** Composite key for a crystal_chunk unit ref. */ -export function toChunkUnitRef(input: { - readonly documentId: string - readonly chunkId: string -}): string { - return `${input.documentId}:${input.chunkId}` -} diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index 8963f95..4e9470f 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -1,9 +1,7 @@ import { sql } from "drizzle-orm"; import { bigint, - doublePrecision, index, - integer, jsonb, pgTable, text, @@ -340,241 +338,3 @@ export const chatMessages = pgTable( export type ChatMessage = typeof chatMessages.$inferSelect; export type NewChatMessage = typeof chatMessages.$inferInsert; - -/** - * Fluid memory: typed insights extracted from human-AI conversation turns - * (as opposed to "crystal memory", which is the parsed document knowledge - * that stays upstream in Knowhere). - * - * One row per extracted insight. `kind` discriminates the typed `payload` - * (see src/domains/memory/types.ts for the payload contract per kind): - * - indicator_pref — a metric the user cares about (name, aliases, - * polarity, importance) - * - stance — a stated position that shapes judgement - * - decision_rule — a when/then rule over indicators - * - entity_of_interest — a company/topic the user tracks - * - * `abstract_l0` / `overview_l1` are the tiered sidecar summaries (L0 = - * one line for pre-filter/dedup context, L1 = short paragraph for later - * cognition injection). L2 is the payload itself. - * - * Lifecycle: rows start `active`; user revisions deactivate rather than - * delete (conservative merge policy), with `version` bumped on merge. - * - * `status` is `active` | `inactive`. `inactive` is not itself a - * disambiguated state — `deactivation_reason` records why an item left - * `active` (e.g. `contradicted`, when distill decides a new turn reverses - * this item). This keeps the decay/lifecycle axis (`status`) separate from - * the reason axis, so an activation-decay job can later flip items to - * `inactive` with a different reason without inventing a new status value. - * - * `source_message_id` points at the assistant message of the turn the - * insight was extracted from; it is set-null on message deletion because - * the insight outlives any single turn. - */ -export const fluidMemoryItems = pgTable( - "fluid_memory_items", - { - id: uuid("id").primaryKey().defaultRandom(), - workspaceId: uuid("workspace_id") - .notNull() - .references(() => workspaces.id, { onDelete: "cascade" }), - kind: text("kind").notNull(), - payload: jsonb("payload").notNull(), - abstractL0: text("abstract_l0").notNull(), - overviewL1: text("overview_l1").notNull(), - sourceMessageId: uuid("source_message_id").references( - () => chatMessages.id, - { onDelete: "set null" }, - ), - confidence: doublePrecision("confidence").notNull(), - status: text("status").notNull(), - deactivationReason: text("deactivation_reason"), - version: integer("version").notNull().default(1), - createdAt: timestamp("created_at", { withTimezone: true }) - .notNull() - .defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }) - .notNull() - .defaultNow(), - }, - (t) => [ - // Workspace lifecycle scans (active vs inactive). - index("fluid_memory_items_workspace_status_idx").on( - t.workspaceId, - t.status, - ), - index("fluid_memory_items_workspace_kind_idx").on(t.workspaceId, t.kind), - ], -); - -export type FluidMemoryItem = typeof fluidMemoryItems.$inferSelect; -export type NewFluidMemoryItem = typeof fluidMemoryItems.$inferInsert; - -/** - * Lexical inverted index over active fluid memory items, used to retrieve - * dedup candidates at extraction time instead of loading the whole memory - * set into the prompt. One row per (item, token); `frequency` counts token - * occurrences in the item's search text. - * - * Invariant: token rows exist iff the owning item is `active`. Writers keep - * this in sync — create inserts rows, merge replaces them, deprecate deletes - * them — so lookups scan tokens alone (no status join) and never surface an - * inactive item. - * - * Tokenization mirrors Knowhere map-nav: single CJK characters plus - * `[a-z0-9_]+` runs. Scoring is idf-weighted token overlap computed in SQL, - * keeping the mechanism on portable Postgres (no pg_trgm/pgvector). - */ -export const fluidMemoryTokens = pgTable( - "fluid_memory_tokens", - { - id: uuid("id").primaryKey().defaultRandom(), - workspaceId: uuid("workspace_id") - .notNull() - .references(() => workspaces.id, { onDelete: "cascade" }), - itemId: uuid("item_id") - .notNull() - .references(() => fluidMemoryItems.id, { onDelete: "cascade" }), - kind: text("kind").notNull(), - token: text("token").notNull(), - frequency: integer("frequency").notNull().default(1), - }, - (t) => [ - // Lookup: candidate tokens within a workspace + kind scope. - index("fluid_memory_tokens_lookup_idx").on( - t.workspaceId, - t.kind, - t.token, - ), - // Rebuild/delete a single item's rows on merge/deprecate. - index("fluid_memory_tokens_item_idx").on(t.itemId), - ], -); - -export type FluidMemoryToken = typeof fluidMemoryTokens.$inferSelect; -export type NewFluidMemoryToken = typeof fluidMemoryTokens.$inferInsert; - -/** - * Append-only audit of extraction decisions, one row per processed turn. - * `operations` is a JSONB array of { op, kind, itemId?, summary, reason? } - * records (op = create | skip | merge | deprecate), mirroring OpenViking's - * memory_diff.json so memory growth stays observable and reversible. - */ -export const memoryDiffs = pgTable( - "memory_diffs", - { - id: uuid("id").primaryKey().defaultRandom(), - workspaceId: uuid("workspace_id") - .notNull() - .references(() => workspaces.id, { onDelete: "cascade" }), - sourceMessageId: uuid("source_message_id").references( - () => chatMessages.id, - { onDelete: "set null" }, - ), - operations: jsonb("operations").notNull(), - createdAt: timestamp("created_at", { withTimezone: true }) - .notNull() - .defaultNow(), - }, - (t) => [ - index("memory_diffs_workspace_created_idx").on(t.workspaceId, t.createdAt), - ], -); - -export type MemoryDiff = typeof memoryDiffs.$inferSelect; -export type NewMemoryDiff = typeof memoryDiffs.$inferInsert; - -/** - * Append-only raw observation layer for fluid memory (L2 evidence). - * - * Each chat turn may write zero or more pending rows here via cheap capture. - * A later distill job consumes a batch, upserts typed items into - * `fluid_memory_items`, and marks these rows `consumed`. Capture never writes - * the distilled layer; distill is the only writer of permanent memory. - * - * Capture stores points of concern only — no early kind classification. - * `subject_hint` is an optional topic anchor for later clustering; distill - * owns the final kind and merge decision. - */ -export const fluidObservations = pgTable( - "fluid_observations", - { - id: uuid("id").primaryKey().defaultRandom(), - workspaceId: uuid("workspace_id") - .notNull() - .references(() => workspaces.id, { onDelete: "cascade" }), - sourceMessageId: uuid("source_message_id").references( - () => chatMessages.id, - { onDelete: "set null" }, - ), - signal: text("signal").notNull(), - evidenceQuote: text("evidence_quote").notNull(), - subjectHint: text("subject_hint"), - referencedDocumentIds: jsonb("referenced_document_ids") - .$type() - .notNull() - .default(sql`'[]'::jsonb`), - confidence: doublePrecision("confidence").notNull(), - status: text("status").notNull().default("pending"), - createdAt: timestamp("created_at", { withTimezone: true }) - .notNull() - .defaultNow(), - consumedAt: timestamp("consumed_at", { withTimezone: true }), - }, - (t) => [ - // Distill scan: pending rows for a workspace in capture order. - index("fluid_observations_workspace_status_created_idx").on( - t.workspaceId, - t.status, - t.createdAt, - ), - ], -); - -export type FluidObservation = typeof fluidObservations.$inferSelect; -export type NewFluidObservation = typeof fluidObservations.$inferInsert; - -/** - * Unified activation ledger for retrievable units, driving time-decay - * importance (see src/domains/retrieval-activation/decay-score.ts). - * - * A "unit" is anything that can be surfaced by retrieval and actually cited - * into an answer: today `fluid_memory` (a `fluid_memory_items` row, keyed by - * its id) and `crystal_chunk` (a Knowhere chunk, which has no local row — - * keyed by `${documentId}:${chunkId}`, composed at write time). - * - * Only "really used in an answer" writes here (a citation), not "entered - * the candidate pool" — this avoids overcounting recall as usage. - * - * For `crystal_chunk`, `created_at` is this row's first-write time (the - * first time Notebook observed this chunk being cited), not the chunk's - * true ingestion time in Knowhere — that timestamp is not available to - * Notebook. This is a known, deliberate approximation. - */ -export const retrievalActivations = pgTable( - "retrieval_activations", - { - id: uuid("id").primaryKey().defaultRandom(), - workspaceId: uuid("workspace_id") - .notNull() - .references(() => workspaces.id, { onDelete: "cascade" }), - unitType: text("unit_type").notNull(), - unitRef: text("unit_ref").notNull(), - activationCount: integer("activation_count").notNull().default(0), - lastActivatedAt: timestamp("last_activated_at", { withTimezone: true }), - createdAt: timestamp("created_at", { withTimezone: true }) - .notNull() - .defaultNow(), - }, - (t) => [ - uniqueIndex("retrieval_activations_unit_idx").on( - t.workspaceId, - t.unitType, - t.unitRef, - ), - ], -); - -export type RetrievalActivation = typeof retrievalActivations.$inferSelect; -export type NewRetrievalActivation = typeof retrievalActivations.$inferInsert; diff --git a/src/integrations/memento/client.ts b/src/integrations/memento/client.ts new file mode 100644 index 0000000..0b0be90 --- /dev/null +++ b/src/integrations/memento/client.ts @@ -0,0 +1,75 @@ +import "server-only" + +import { + FetchHttpClient, + HttpClient, + HttpClientRequest, +} from "@effect/platform" +import { Effect } from "effect" + +import { summarizeUnknownError } from "@/lib/format-log-value" +import { logger } from "@/lib/logger" +import { readMementoConfig } from "./config" + +export type MementoCaptureInput = { + readonly workspaceId: string + readonly sourceMessageId: string | null + readonly userText: string + readonly assistantText: string + readonly referencedDocumentIds: readonly string[] +} + +export type MementoActivationInput = { + readonly workspaceId: string + readonly unitType: "fluid_memory" | "crystal_chunk" + readonly unitRef: string +} + +export async function captureMemoryTurn( + input: MementoCaptureInput, +): Promise { + try { + await postJson("/turns/capture", input) + } catch (error) { + logger.warn("chat: failed to capture memory turn", { + workspaceId: input.workspaceId, + error: summarizeUnknownError(error), + }) + } +} + +export async function recordActivations( + activations: readonly MementoActivationInput[], +): Promise { + if (activations.length === 0) return + + try { + await postJson("/activations", { activations }) + } catch (error) { + logger.warn("chat: failed to record activations", { + workspaceId: activations[0]?.workspaceId, + count: activations.length, + error: summarizeUnknownError(error), + }) + } +} + +async function postJson(path: string, body: unknown): Promise { + const status = await Effect.runPromise( + Effect.gen(function* () { + const { baseUrl, serviceKey } = readMementoConfig() + const request = yield* HttpClientRequest.post(`${baseUrl}${path}`).pipe( + HttpClientRequest.setHeader( + "Authorization", + `Bearer ${serviceKey}`, + ), + HttpClientRequest.bodyJson(body), + ) + const response = yield* HttpClient.execute(request) + return response.status + }).pipe(Effect.provide(FetchHttpClient.layer)), + ) + if (status < 200 || status >= 300) { + throw new Error(`memento ${path}: HTTP ${status}`) + } +} diff --git a/src/integrations/memento/config.ts b/src/integrations/memento/config.ts new file mode 100644 index 0000000..29eb234 --- /dev/null +++ b/src/integrations/memento/config.ts @@ -0,0 +1,18 @@ +import "server-only" + +export type MementoConfig = { + readonly baseUrl: string + readonly serviceKey: string +} + +export function readMementoConfig(): MementoConfig { + const baseUrl = process.env.MEMENTO_BASE_URL + const serviceKey = process.env.MEMENTO_SERVICE_KEY + if (!baseUrl) { + throw new Error("MEMENTO_BASE_URL is required.") + } + if (!serviceKey) { + throw new Error("MEMENTO_SERVICE_KEY is required.") + } + return { baseUrl, serviceKey } +} diff --git a/src/integrations/memento/memory-tools.ts b/src/integrations/memento/memory-tools.ts new file mode 100644 index 0000000..052f160 --- /dev/null +++ b/src/integrations/memento/memory-tools.ts @@ -0,0 +1,105 @@ +import "server-only" + +import { Client } from "@modelcontextprotocol/sdk/client/index.js" +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js" + +import type { + MemorySearchItem, + MemorySearchKind, + MemorySearchRequest, + MemorySearchResponse, + MemoryToolRuntime, +} from "@/agent-harness" +import { memorySearchKinds } from "@/agent-harness" +import { readMementoConfig } from "./config" + +type MementoMemoryToolsInput = { + readonly workspaceId: string +} + +export const mementoMemoryTools = { + createRuntime(input: MementoMemoryToolsInput): MemoryToolRuntime { + return { + search: (request) => searchWorkspaceMemory(input.workspaceId, request), + } + }, +} as const + +async function searchWorkspaceMemory( + workspaceId: string, + request: MemorySearchRequest, +): Promise { + const { baseUrl, serviceKey } = readMementoConfig() + const transport = new StreamableHTTPClientTransport(new URL(`${baseUrl}/mcp`), { + requestInit: { + headers: { + Authorization: `Bearer ${serviceKey}`, + }, + }, + }) + const client = new Client({ + name: "knowhere-notebook", + version: "0.1.0", + }) + await client.connect(transport) + try { + const result = await client.callTool({ + name: "memory_search", + arguments: { + workspaceId, + query: request.query, + ...(request.kinds ? { kinds: request.kinds } : {}), + }, + }) + if (result.isError) { + throw new Error("memento memory_search failed") + } + return parseMemorySearchResponse(result.structuredContent) + } finally { + await client.close() + } +} + +function parseMemorySearchResponse(value: unknown): MemorySearchResponse { + if (!value || typeof value !== "object") { + throw new Error("memento memory_search returned no structured content") + } + const record = value as Record + if (typeof record.query !== "string" || !Array.isArray(record.items)) { + throw new Error("memento memory_search structured content is invalid") + } + return { + query: record.query, + items: record.items.map(parseMemorySearchItem), + } +} + +function parseMemorySearchItem(value: unknown): MemorySearchItem { + if (!value || typeof value !== "object") { + throw new Error("memento memory_search item is invalid") + } + const item = value as Record + if ( + typeof item.ref !== "string" || + typeof item.itemId !== "string" || + typeof item.abstractL0 !== "string" || + typeof item.overviewL1 !== "string" || + !isMemorySearchKind(item.kind) + ) { + throw new Error("memento memory_search item is invalid") + } + return { + ref: item.ref, + itemId: item.itemId, + kind: item.kind, + abstractL0: item.abstractL0, + overviewL1: item.overviewL1, + } +} + +function isMemorySearchKind(value: unknown): value is MemorySearchKind { + return ( + typeof value === "string" && + (memorySearchKinds as readonly string[]).includes(value) + ) +} From a5c6b0f22ed236c21bcfb181ff8811d8341a52df Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Thu, 10 Sep 2026 17:41:47 +0800 Subject: [PATCH 4/4] refactor(chat): update namespace handling and improve Knowhere client configuration - Changed the namespace from "notebook-namespace" to "default" in chat service tests to align with new defaults. - Updated the chat service to use a shared library namespace instead of fetching compatible namespaces. - Increased the timeout for the Knowhere client to 120 seconds to enhance reliability during API calls. These changes streamline namespace management and improve the integration with the Knowhere API. --- src/domains/chat/service.test.ts | 4 ++-- src/domains/chat/service.ts | 4 ++-- src/integrations/knowhere.test.ts | 1 + src/integrations/knowhere.ts | 1 + 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/domains/chat/service.test.ts b/src/domains/chat/service.test.ts index 133e0a5..4aa1a5a 100644 --- a/src/domains/chat/service.test.ts +++ b/src/domains/chat/service.test.ts @@ -54,7 +54,7 @@ describe("handleChatTurn", () => { }); } expect(retrieval.query).toHaveBeenCalledWith({ - namespace: "notebook-namespace", + namespace: "default", query: "What does the document say?", topK: 8, useAgentic: true, @@ -245,7 +245,7 @@ describe("handleChatTurn", () => { knowhereTools: expect.any(Object), }); expect(retrieval.query).toHaveBeenCalledWith({ - namespace: "notebook-namespace", + namespace: "default", query: "Tesla Q4 2025 Update energy generation and storage deployments", topK: 8, useAgentic: true, diff --git a/src/domains/chat/service.ts b/src/domains/chat/service.ts index 23bfa68..7839ab0 100644 --- a/src/domains/chat/service.ts +++ b/src/domains/chat/service.ts @@ -9,7 +9,7 @@ import { } from "." import { toChatMessageView } from "./view" import type { ChatMessage, ChatThread, Source, Workspace } from "@/infrastructure/db/schema" -import { getCompatibleNamespaces } from "@/domains/sources/namespace" +import { sharedLibraryNamespace } from "@/domains/sources/namespace" import type { ChatArtifactView, ChatCitationView, @@ -125,7 +125,7 @@ export const handleChatTurnEffect = (input: ChatTurnInput) => const answer = yield* answerQuestionWithRetrieval({ question: input.question, namespace: input.workspace.namespace, - namespaces: getCompatibleNamespaces(input.workspace), + namespaces: [sharedLibraryNamespace], sources: readySources, useAgentic: input.useAgentic ?? true, excludedSourceIds: input.excludedSourceIds, diff --git a/src/integrations/knowhere.test.ts b/src/integrations/knowhere.test.ts index 74b8c25..a2c4845 100644 --- a/src/integrations/knowhere.test.ts +++ b/src/integrations/knowhere.test.ts @@ -58,6 +58,7 @@ describe("makeKnowhereClient", () => { expect(constructorSpy).toHaveBeenCalledWith({ apiKey: "sk_test", baseURL: "https://api-staging.knowhereto.ai", + timeout: 120_000, }); }); diff --git a/src/integrations/knowhere.ts b/src/integrations/knowhere.ts index e55fc0b..a1ca2ed 100644 --- a/src/integrations/knowhere.ts +++ b/src/integrations/knowhere.ts @@ -37,6 +37,7 @@ export function makeKnowhereClient(apiKey: string): Knowhere { const options: ConstructorParameters[0] = { apiKey, baseURL: process.env.KNOWHERE_BASE_URL, + timeout: 120_000, } const client = new Knowhere(options) return wrapKnowhereClient(client)