diff --git a/.env.local.example b/.env.local.example index 13078722..097692fe 100644 --- a/.env.local.example +++ b/.env.local.example @@ -1,11 +1,17 @@ -# Optional API base URL. Defaults to production when unset. -# Use staging when validating staging keys. -# KNOWHERE_BASE_URL=https://api-staging.knowhereto.ai +# Knowhere API origin. Unset also means production. +# Local recommendation: production. Staging is only for staging keys. +KNOWHERE_BASE_URL=https://api.knowhereto.ai -# Optional development override. When set, Notebook skips Dashboard session +# Local development override. When set, Notebook skips Dashboard session # auth and Dashboard-issued JWT creation, then calls Knowhere directly with # this key. Leave unset for production and Dashboard-authenticated staging. -# KNOWHERE_API_KEY=sk_your_development_key_here +# KNOWHERE_API_KEY= + +# --- Local demo case only (观心). Not Notebook core. --- +# Put the real key in gitignored `.env.local`, copied from +# `观心2.0-RAG-v1.1-demo/_retrieval/.env`. +# Do not copy that file's KNOWHERE_BASE_URL — it is a retrieval endpoint +# path, not the SDK origin above. # --- Chat provider (server-side only) --- # Vercel AI Gateway key; AI SDK picks it up automatically @@ -61,3 +67,8 @@ DATABASE_URL=postgres://user:password@host/db # pg — postgres-js (use for local dev against a plain Postgres, # or for AWS Aurora Postgres if/when we migrate off Neon) DATABASE_DRIVER=pg + +# --- Memento (fluid memory + retrieval activation) --- +# Standalone REST + MCP service. Notebook no longer owns these tables. +MEMENTO_BASE_URL=http://localhost:8787 +MEMENTO_SERVICE_KEY= diff --git a/.pnpm-store/v11/index.db b/.pnpm-store/v11/index.db new file mode 100644 index 00000000..8fdf9e7d Binary files /dev/null and b/.pnpm-store/v11/index.db differ diff --git a/.repos/OpenViking b/.repos/OpenViking new file mode 160000 index 00000000..f6d9dec6 --- /dev/null +++ b/.repos/OpenViking @@ -0,0 +1 @@ +Subproject commit f6d9dec6b6ae16a152c437fd4ad81ca45fcc8648 diff --git a/drizzle/0015_bumpy_vulcan.sql b/drizzle/0015_bumpy_vulcan.sql new file mode 100644 index 00000000..2b383759 --- /dev/null +++ b/drizzle/0015_bumpy_vulcan.sql @@ -0,0 +1,17 @@ +CREATE TABLE "fluid_observations" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "source_message_id" uuid, + "signal" text NOT NULL, + "evidence_quote" text NOT NULL, + "subject_hint" text, + "referenced_document_ids" jsonb DEFAULT '[]'::jsonb NOT NULL, + "confidence" double precision NOT NULL, + "status" text DEFAULT 'pending' NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "consumed_at" timestamp with time zone +); +--> statement-breakpoint +ALTER TABLE "fluid_observations" ADD CONSTRAINT "fluid_observations_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "fluid_observations" ADD CONSTRAINT "fluid_observations_source_message_id_chat_messages_id_fk" FOREIGN KEY ("source_message_id") REFERENCES "public"."chat_messages"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "fluid_observations_workspace_status_created_idx" ON "fluid_observations" USING btree ("workspace_id","status","created_at"); \ No newline at end of file diff --git a/drizzle/0016_wealthy_matthew_murdock.sql b/drizzle/0016_wealthy_matthew_murdock.sql new file mode 100644 index 00000000..4d7a57ec --- /dev/null +++ b/drizzle/0016_wealthy_matthew_murdock.sql @@ -0,0 +1,13 @@ +CREATE TABLE "retrieval_activations" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "workspace_id" uuid NOT NULL, + "unit_type" text NOT NULL, + "unit_ref" text NOT NULL, + "activation_count" integer DEFAULT 0 NOT NULL, + "last_activated_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "fluid_memory_items" ADD COLUMN "deactivation_reason" text;--> statement-breakpoint +ALTER TABLE "retrieval_activations" ADD CONSTRAINT "retrieval_activations_workspace_id_workspaces_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."workspaces"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "retrieval_activations_unit_idx" ON "retrieval_activations" USING btree ("workspace_id","unit_type","unit_ref"); \ No newline at end of file diff --git a/drizzle/0017_overjoyed_shooting_star.sql b/drizzle/0017_overjoyed_shooting_star.sql new file mode 100644 index 00000000..6d97d3ac --- /dev/null +++ b/drizzle/0017_overjoyed_shooting_star.sql @@ -0,0 +1,5 @@ +DROP TABLE "fluid_memory_items" CASCADE;--> statement-breakpoint +DROP TABLE "fluid_memory_tokens" CASCADE;--> statement-breakpoint +DROP TABLE "fluid_observations" CASCADE;--> statement-breakpoint +DROP TABLE "memory_diffs" CASCADE;--> statement-breakpoint +DROP TABLE "retrieval_activations" CASCADE; \ No newline at end of file diff --git a/drizzle/meta/0015_snapshot.json b/drizzle/meta/0015_snapshot.json new file mode 100644 index 00000000..3b313fd5 --- /dev/null +++ b/drizzle/meta/0015_snapshot.json @@ -0,0 +1,1444 @@ +{ + "id": "be8deb1f-3ec8-4304-85d0-41403629970c", + "prevId": "72bfb4fe-4048-4dce-a5fa-ce8eea880aa3", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_memory_items": { + "name": "fluid_memory_items", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "abstract_l0": { + "name": "abstract_l0", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "overview_l1": { + "name": "overview_l1", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "fluid_memory_items_workspace_status_idx": { + "name": "fluid_memory_items_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fluid_memory_items_workspace_kind_idx": { + "name": "fluid_memory_items_workspace_kind_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_memory_items_workspace_id_workspaces_id_fk": { + "name": "fluid_memory_items_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_memory_items_source_message_id_chat_messages_id_fk": { + "name": "fluid_memory_items_source_message_id_chat_messages_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_memory_tokens": { + "name": "fluid_memory_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "item_id": { + "name": "item_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "frequency": { + "name": "frequency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + } + }, + "indexes": { + "fluid_memory_tokens_lookup_idx": { + "name": "fluid_memory_tokens_lookup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fluid_memory_tokens_item_idx": { + "name": "fluid_memory_tokens_item_idx", + "columns": [ + { + "expression": "item_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_memory_tokens_workspace_id_workspaces_id_fk": { + "name": "fluid_memory_tokens_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_memory_tokens", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_memory_tokens_item_id_fluid_memory_items_id_fk": { + "name": "fluid_memory_tokens_item_id_fluid_memory_items_id_fk", + "tableFrom": "fluid_memory_tokens", + "tableTo": "fluid_memory_items", + "columnsFrom": [ + "item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_observations": { + "name": "fluid_observations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "signal": { + "name": "signal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "evidence_quote": { + "name": "evidence_quote", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "subject_hint": { + "name": "subject_hint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "referenced_document_ids": { + "name": "referenced_document_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "confidence": { + "name": "confidence", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "fluid_observations_workspace_status_created_idx": { + "name": "fluid_observations_workspace_status_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_observations_workspace_id_workspaces_id_fk": { + "name": "fluid_observations_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_observations", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_observations_source_message_id_chat_messages_id_fk": { + "name": "fluid_observations_source_message_id_chat_messages_id_fk", + "tableFrom": "fluid_observations", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_diffs": { + "name": "memory_diffs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memory_diffs_workspace_created_idx": { + "name": "memory_diffs_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_diffs_workspace_id_workspaces_id_fk": { + "name": "memory_diffs_workspace_id_workspaces_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memory_diffs_source_message_id_chat_messages_id_fk": { + "name": "memory_diffs_source_message_id_chat_messages_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parsed_document_sync_leases": { + "name": "parsed_document_sync_leases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_token": { + "name": "lease_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_reason": { + "name": "release_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parsed_document_sync_leases_token_idx": { + "name": "parsed_document_sync_leases_token_idx", + "columns": [ + { + "expression": "lease_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_active_idx": { + "name": "parsed_document_sync_leases_active_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_workspace_active_idx": { + "name": "parsed_document_sync_leases_workspace_active_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_document_active_idx": { + "name": "parsed_document_sync_leases_document_active_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parsed_document_sync_leases_workspace_id_workspaces_id_fk": { + "name": "parsed_document_sync_leases_workspace_id_workspaces_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parsed_document_sync_leases_source_id_sources_id_fk": { + "name": "parsed_document_sync_leases_source_id_sources_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/0016_snapshot.json b/drizzle/meta/0016_snapshot.json new file mode 100644 index 00000000..77ef531c --- /dev/null +++ b/drizzle/meta/0016_snapshot.json @@ -0,0 +1,1550 @@ +{ + "id": "55d73dc7-9826-4cb5-9bb6-3506b8fa338a", + "prevId": "be8deb1f-3ec8-4304-85d0-41403629970c", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_memory_items": { + "name": "fluid_memory_items", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "abstract_l0": { + "name": "abstract_l0", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "overview_l1": { + "name": "overview_l1", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deactivation_reason": { + "name": "deactivation_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "fluid_memory_items_workspace_status_idx": { + "name": "fluid_memory_items_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fluid_memory_items_workspace_kind_idx": { + "name": "fluid_memory_items_workspace_kind_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_memory_items_workspace_id_workspaces_id_fk": { + "name": "fluid_memory_items_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_memory_items_source_message_id_chat_messages_id_fk": { + "name": "fluid_memory_items_source_message_id_chat_messages_id_fk", + "tableFrom": "fluid_memory_items", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_memory_tokens": { + "name": "fluid_memory_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "item_id": { + "name": "item_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "frequency": { + "name": "frequency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + } + }, + "indexes": { + "fluid_memory_tokens_lookup_idx": { + "name": "fluid_memory_tokens_lookup_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fluid_memory_tokens_item_idx": { + "name": "fluid_memory_tokens_item_idx", + "columns": [ + { + "expression": "item_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_memory_tokens_workspace_id_workspaces_id_fk": { + "name": "fluid_memory_tokens_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_memory_tokens", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_memory_tokens_item_id_fluid_memory_items_id_fk": { + "name": "fluid_memory_tokens_item_id_fluid_memory_items_id_fk", + "tableFrom": "fluid_memory_tokens", + "tableTo": "fluid_memory_items", + "columnsFrom": [ + "item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fluid_observations": { + "name": "fluid_observations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "signal": { + "name": "signal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "evidence_quote": { + "name": "evidence_quote", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "subject_hint": { + "name": "subject_hint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "referenced_document_ids": { + "name": "referenced_document_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "confidence": { + "name": "confidence", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "fluid_observations_workspace_status_created_idx": { + "name": "fluid_observations_workspace_status_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fluid_observations_workspace_id_workspaces_id_fk": { + "name": "fluid_observations_workspace_id_workspaces_id_fk", + "tableFrom": "fluid_observations", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "fluid_observations_source_message_id_chat_messages_id_fk": { + "name": "fluid_observations_source_message_id_chat_messages_id_fk", + "tableFrom": "fluid_observations", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memory_diffs": { + "name": "memory_diffs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_message_id": { + "name": "source_message_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memory_diffs_workspace_created_idx": { + "name": "memory_diffs_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memory_diffs_workspace_id_workspaces_id_fk": { + "name": "memory_diffs_workspace_id_workspaces_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memory_diffs_source_message_id_chat_messages_id_fk": { + "name": "memory_diffs_source_message_id_chat_messages_id_fk", + "tableFrom": "memory_diffs", + "tableTo": "chat_messages", + "columnsFrom": [ + "source_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parsed_document_sync_leases": { + "name": "parsed_document_sync_leases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_token": { + "name": "lease_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_reason": { + "name": "release_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parsed_document_sync_leases_token_idx": { + "name": "parsed_document_sync_leases_token_idx", + "columns": [ + { + "expression": "lease_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_active_idx": { + "name": "parsed_document_sync_leases_active_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_workspace_active_idx": { + "name": "parsed_document_sync_leases_workspace_active_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_document_active_idx": { + "name": "parsed_document_sync_leases_document_active_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parsed_document_sync_leases_workspace_id_workspaces_id_fk": { + "name": "parsed_document_sync_leases_workspace_id_workspaces_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parsed_document_sync_leases_source_id_sources_id_fk": { + "name": "parsed_document_sync_leases_source_id_sources_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.retrieval_activations": { + "name": "retrieval_activations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "unit_type": { + "name": "unit_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "unit_ref": { + "name": "unit_ref", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activation_count": { + "name": "activation_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_activated_at": { + "name": "last_activated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "retrieval_activations_unit_idx": { + "name": "retrieval_activations_unit_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "unit_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "unit_ref", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "retrieval_activations_workspace_id_workspaces_id_fk": { + "name": "retrieval_activations_workspace_id_workspaces_id_fk", + "tableFrom": "retrieval_activations", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/0017_snapshot.json b/drizzle/meta/0017_snapshot.json new file mode 100644 index 00000000..52d88234 --- /dev/null +++ b/drizzle/meta/0017_snapshot.json @@ -0,0 +1,932 @@ +{ + "id": "d4de5ce8-e16b-4dc2-a9ad-2b76bbca8150", + "prevId": "55d73dc7-9826-4cb5-9bb6-3506b8fa338a", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parsed_document_sync_leases": { + "name": "parsed_document_sync_leases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_token": { + "name": "lease_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_reason": { + "name": "release_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parsed_document_sync_leases_token_idx": { + "name": "parsed_document_sync_leases_token_idx", + "columns": [ + { + "expression": "lease_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_active_idx": { + "name": "parsed_document_sync_leases_active_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_workspace_active_idx": { + "name": "parsed_document_sync_leases_workspace_active_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_document_active_idx": { + "name": "parsed_document_sync_leases_document_active_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parsed_document_sync_leases_workspace_id_workspaces_id_fk": { + "name": "parsed_document_sync_leases_workspace_id_workspaces_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parsed_document_sync_leases_source_id_sources_id_fk": { + "name": "parsed_document_sync_leases_source_id_sources_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index b9be4f54..4ee28af6 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -106,6 +106,27 @@ "when": 1788252035435, "tag": "0014_messy_apocalypse", "breakpoints": true + }, + { + "idx": 15, + "version": "7", + "when": 1788418416499, + "tag": "0015_bumpy_vulcan", + "breakpoints": true + }, + { + "idx": 16, + "version": "7", + "when": 1788952172906, + "tag": "0016_wealthy_matthew_murdock", + "breakpoints": true + }, + { + "idx": 17, + "version": "7", + "when": 1789017481605, + "tag": "0017_overjoyed_shooting_star", + "breakpoints": true } ] } \ No newline at end of file diff --git a/next.config.ts b/next.config.ts index b73f5201..cd1fac0a 100644 --- a/next.config.ts +++ b/next.config.ts @@ -10,6 +10,7 @@ const nextConfig: NextConfig = { "@ontos-ai/knowhere-sdk", "@napi-rs/canvas", "piscina", + "@modelcontextprotocol/sdk", ], allowedDevOrigins: [ "127.0.0.1", diff --git a/package.json b/package.json index f3778893..4ab25322 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "@ai-sdk/react": "^3.0.177", "@antv/chart-visualization-skills": "0.1.3", "@effect/platform": "^0.96.1", + "@modelcontextprotocol/sdk": "^1.30.0", "@napi-rs/canvas": "^1.0.2", "@neondatabase/serverless": "^1.1.0", "@ontos-ai/knowhere-sdk": "^2.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8847510a..d4db84c2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: '@effect/platform': specifier: ^0.96.1 version: 0.96.1(effect@3.21.2) + '@modelcontextprotocol/sdk': + specifier: ^1.30.0 + version: 1.30.0(supports-color@7.2.0)(zod@4.4.3) '@napi-rs/canvas': specifier: ^1.0.2 version: 1.0.2 @@ -25,7 +28,7 @@ importers: version: 1.1.0 '@ontos-ai/knowhere-sdk': specifier: ^2.2.0 - version: 2.2.0 + version: 2.2.0(debug@4.4.3(supports-color@7.2.0))(supports-color@7.2.0) '@radix-ui/react-alert-dialog': specifier: ^1.1.15 version: 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -97,7 +100,7 @@ importers: version: 1.12.0 next: specifier: 16.2.4 - version: 16.2.4(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + version: 16.2.4(@babel/core@7.29.0(supports-color@7.2.0))(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) next-themes: specifier: ^0.4.6 version: 0.4.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -121,13 +124,13 @@ importers: version: 19.2.4(react@19.2.4) react-markdown: specifier: ^10.1.0 - version: 10.1.0(@types/react@19.2.14)(react@19.2.4) + version: 10.1.0(@types/react@19.2.14)(react@19.2.4)(supports-color@7.2.0) react-pdf: specifier: ^10.4.1 version: 10.4.1(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) remark-gfm: specifier: ^4.0.1 - version: 4.0.1 + version: 4.0.1(supports-color@7.2.0) server-only: specifier: ^0.0.1 version: 0.0.1 @@ -182,16 +185,16 @@ importers: version: 0.31.10 eslint: specifier: ^9 - version: 9.39.4(jiti@2.7.0) + version: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) eslint-config-next: specifier: 16.2.4 - version: 16.2.4(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) + version: 16.2.4(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) jsdom: specifier: ^29.1.1 version: 29.1.1(@noble/hashes@1.8.0) shadcn: specifier: ^4.7.0 - version: 4.7.0(@types/node@20.19.39)(typescript@6.0.3) + version: 4.7.0(@types/node@20.19.39)(supports-color@7.2.0)(typescript@6.0.3) tailwindcss: specifier: ^4 version: 4.2.4 @@ -455,11 +458,11 @@ packages: '@esbuild-kit/core-utils@3.3.2': resolution: {integrity: sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild-kit/esm-loader@2.6.5': resolution: {integrity: sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==} - deprecated: 'Merged into tsx: https://tsx.is' + deprecated: 'Merged into tsx: https://tsx.hirok.io' '@esbuild/aix-ppc64@0.25.12': resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} @@ -1197,8 +1200,8 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} - '@modelcontextprotocol/sdk@1.29.0': - resolution: {integrity: sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==} + '@modelcontextprotocol/sdk@1.30.0': + resolution: {integrity: sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA==} engines: {node: '>=18'} peerDependencies: '@cfworker/json-schema': ^4.1.1 @@ -2594,6 +2597,7 @@ packages: '@xmldom/xmldom@0.8.13': resolution: {integrity: sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==} engines: {node: '>=10.0.0'} + deprecated: this version has critical issues, please update to the latest version accepts@2.0.0: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} @@ -2948,6 +2952,7 @@ packages: crypto-js@4.2.0: resolution: {integrity: sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==} + deprecated: Active development of CryptoJS has been discontinued. This library is no longer maintained. css-tree@3.2.1: resolution: {integrity: sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==} @@ -3394,6 +3399,7 @@ packages: eslint@9.39.4: resolution: {integrity: sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. hasBin: true peerDependencies: jiti: '*' @@ -5733,20 +5739,20 @@ snapshots: '@babel/compat-data@7.29.3': {} - '@babel/core@7.29.0': + '@babel/core@7.29.0(supports-color@7.2.0)': dependencies: '@babel/code-frame': 7.29.0 '@babel/generator': 7.29.1 '@babel/helper-compilation-targets': 7.28.6 - '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0) + '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helpers': 7.29.2 '@babel/parser': 7.29.3 '@babel/template': 7.28.6 - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) '@babel/types': 7.29.0 '@jridgewell/remapping': 2.3.5 convert-source-map: 2.0.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -5773,41 +5779,41 @@ snapshots: lru-cache: 5.1.1 semver: 6.3.1 - '@babel/helper-create-class-features-plugin@7.29.3(@babel/core@7.29.0)': + '@babel/helper-create-class-features-plugin@7.29.3(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/helper-annotate-as-pure': 7.27.3 - '@babel/helper-member-expression-to-functions': 7.28.5 + '@babel/helper-member-expression-to-functions': 7.28.5(supports-color@7.2.0) '@babel/helper-optimise-call-expression': 7.27.1 - '@babel/helper-replace-supers': 7.28.6(@babel/core@7.29.0) - '@babel/helper-skip-transparent-expression-wrappers': 7.27.1 - '@babel/traverse': 7.29.0 + '@babel/helper-replace-supers': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/helper-skip-transparent-expression-wrappers': 7.27.1(supports-color@7.2.0) + '@babel/traverse': 7.29.0(supports-color@7.2.0) semver: 6.3.1 transitivePeerDependencies: - supports-color '@babel/helper-globals@7.28.0': {} - '@babel/helper-member-expression-to-functions@7.28.5': + '@babel/helper-member-expression-to-functions@7.28.5(supports-color@7.2.0)': dependencies: - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) '@babel/types': 7.29.0 transitivePeerDependencies: - supports-color - '@babel/helper-module-imports@7.28.6': + '@babel/helper-module-imports@7.28.6(supports-color@7.2.0)': dependencies: - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) '@babel/types': 7.29.0 transitivePeerDependencies: - supports-color - '@babel/helper-module-transforms@7.28.6(@babel/core@7.29.0)': + '@babel/helper-module-transforms@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 - '@babel/helper-module-imports': 7.28.6 + '@babel/core': 7.29.0(supports-color@7.2.0) + '@babel/helper-module-imports': 7.28.6(supports-color@7.2.0) '@babel/helper-validator-identifier': 7.28.5 - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -5817,18 +5823,18 @@ snapshots: '@babel/helper-plugin-utils@7.28.6': {} - '@babel/helper-replace-supers@7.28.6(@babel/core@7.29.0)': + '@babel/helper-replace-supers@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 - '@babel/helper-member-expression-to-functions': 7.28.5 + '@babel/core': 7.29.0(supports-color@7.2.0) + '@babel/helper-member-expression-to-functions': 7.28.5(supports-color@7.2.0) '@babel/helper-optimise-call-expression': 7.27.1 - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) transitivePeerDependencies: - supports-color - '@babel/helper-skip-transparent-expression-wrappers@7.27.1': + '@babel/helper-skip-transparent-expression-wrappers@7.27.1(supports-color@7.2.0)': dependencies: - '@babel/traverse': 7.29.0 + '@babel/traverse': 7.29.0(supports-color@7.2.0) '@babel/types': 7.29.0 transitivePeerDependencies: - supports-color @@ -5848,43 +5854,43 @@ snapshots: dependencies: '@babel/types': 7.29.0 - '@babel/plugin-syntax-jsx@7.28.6(@babel/core@7.29.0)': + '@babel/plugin-syntax-jsx@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))': dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.28.6 - '@babel/plugin-syntax-typescript@7.28.6(@babel/core@7.29.0)': + '@babel/plugin-syntax-typescript@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))': dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.28.6 - '@babel/plugin-transform-modules-commonjs@7.28.6(@babel/core@7.29.0)': + '@babel/plugin-transform-modules-commonjs@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 - '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0) + '@babel/core': 7.29.0(supports-color@7.2.0) + '@babel/helper-module-transforms': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.28.6 transitivePeerDependencies: - supports-color - '@babel/plugin-transform-typescript@7.28.6(@babel/core@7.29.0)': + '@babel/plugin-transform-typescript@7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/helper-annotate-as-pure': 7.27.3 - '@babel/helper-create-class-features-plugin': 7.29.3(@babel/core@7.29.0) + '@babel/helper-create-class-features-plugin': 7.29.3(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.28.6 - '@babel/helper-skip-transparent-expression-wrappers': 7.27.1 - '@babel/plugin-syntax-typescript': 7.28.6(@babel/core@7.29.0) + '@babel/helper-skip-transparent-expression-wrappers': 7.27.1(supports-color@7.2.0) + '@babel/plugin-syntax-typescript': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0)) transitivePeerDependencies: - supports-color - '@babel/preset-typescript@7.28.5(@babel/core@7.29.0)': + '@babel/preset-typescript@7.28.5(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/helper-plugin-utils': 7.28.6 '@babel/helper-validator-option': 7.27.1 - '@babel/plugin-syntax-jsx': 7.28.6(@babel/core@7.29.0) - '@babel/plugin-transform-modules-commonjs': 7.28.6(@babel/core@7.29.0) - '@babel/plugin-transform-typescript': 7.28.6(@babel/core@7.29.0) + '@babel/plugin-syntax-jsx': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0)) + '@babel/plugin-transform-modules-commonjs': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/plugin-transform-typescript': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -5896,7 +5902,7 @@ snapshots: '@babel/parser': 7.29.3 '@babel/types': 7.29.0 - '@babel/traverse@7.29.0': + '@babel/traverse@7.29.0(supports-color@7.2.0)': dependencies: '@babel/code-frame': 7.29.0 '@babel/generator': 7.29.1 @@ -5904,7 +5910,7 @@ snapshots: '@babel/parser': 7.29.3 '@babel/template': 7.28.6 '@babel/types': 7.29.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -6217,17 +6223,17 @@ snapshots: '@esbuild/win32-x64@0.27.7': optional: true - '@eslint-community/eslint-utils@4.9.1(eslint@9.39.4(jiti@2.7.0))': + '@eslint-community/eslint-utils@4.9.1(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))': dependencies: - eslint: 9.39.4(jiti@2.7.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) eslint-visitor-keys: 3.4.3 '@eslint-community/regexpp@4.12.2': {} - '@eslint/config-array@0.21.2': + '@eslint/config-array@0.21.2(supports-color@7.2.0)': dependencies: '@eslint/object-schema': 2.1.7 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) minimatch: 3.1.5 transitivePeerDependencies: - supports-color @@ -6240,10 +6246,10 @@ snapshots: dependencies: '@types/json-schema': 7.0.15 - '@eslint/eslintrc@3.3.5': + '@eslint/eslintrc@3.3.5(supports-color@7.2.0)': dependencies: ajv: 6.15.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) espree: 10.4.0 globals: 14.0.0 ignore: 5.3.2 @@ -6447,7 +6453,7 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 - '@modelcontextprotocol/sdk@1.29.0(zod@3.25.76)': + '@modelcontextprotocol/sdk@1.30.0(supports-color@7.2.0)(zod@3.25.76)': dependencies: '@hono/node-server': 1.19.14(hono@4.12.17) ajv: 8.20.0 @@ -6457,8 +6463,8 @@ snapshots: cross-spawn: 7.0.6 eventsource: 3.0.7 eventsource-parser: 3.0.8 - express: 5.2.1 - express-rate-limit: 8.5.0(express@5.2.1) + express: 5.2.1(supports-color@7.2.0) + express-rate-limit: 8.5.0(express@5.2.1(supports-color@7.2.0)) hono: 4.12.17 jose: 6.2.3 json-schema-typed: 8.0.2 @@ -6469,6 +6475,28 @@ snapshots: transitivePeerDependencies: - supports-color + '@modelcontextprotocol/sdk@1.30.0(supports-color@7.2.0)(zod@4.4.3)': + dependencies: + '@hono/node-server': 1.19.14(hono@4.12.17) + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) + content-type: 1.0.5 + cors: 2.8.6 + cross-spawn: 7.0.6 + eventsource: 3.0.7 + eventsource-parser: 3.0.8 + express: 5.2.1(supports-color@7.2.0) + express-rate-limit: 8.5.0(express@5.2.1(supports-color@7.2.0)) + hono: 4.12.17 + jose: 6.2.3 + json-schema-typed: 8.0.2 + pkce-challenge: 5.0.1 + raw-body: 3.0.2 + zod: 4.4.3 + zod-to-json-schema: 3.25.2(zod@4.4.3) + transitivePeerDependencies: + - supports-color + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': optional: true @@ -6731,9 +6759,9 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} - '@ontos-ai/knowhere-sdk@2.2.0': + '@ontos-ai/knowhere-sdk@2.2.0(debug@4.4.3(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: - axios: 1.18.1 + axios: 1.18.1(debug@4.4.3(supports-color@7.2.0))(supports-color@7.2.0) jszip: 3.10.1 transitivePeerDependencies: - debug @@ -7400,15 +7428,15 @@ snapshots: '@types/validate-npm-package-name@4.0.2': {} - '@typescript-eslint/eslint-plugin@8.59.2(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/eslint-plugin@8.59.2(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3)': dependencies: '@eslint-community/regexpp': 4.12.2 - '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) '@typescript-eslint/scope-manager': 8.59.2 - '@typescript-eslint/type-utils': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/type-utils': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) '@typescript-eslint/visitor-keys': 8.59.2 - eslint: 9.39.4(jiti@2.7.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) ignore: 7.0.5 natural-compare: 1.4.0 ts-api-utils: 2.5.0(typescript@6.0.3) @@ -7416,23 +7444,23 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3)': dependencies: '@typescript-eslint/scope-manager': 8.59.2 '@typescript-eslint/types': 8.59.2 - '@typescript-eslint/typescript-estree': 8.59.2(typescript@6.0.3) + '@typescript-eslint/typescript-estree': 8.59.2(supports-color@7.2.0)(typescript@6.0.3) '@typescript-eslint/visitor-keys': 8.59.2 - debug: 4.4.3 - eslint: 9.39.4(jiti@2.7.0) + debug: 4.4.3(supports-color@7.2.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color - '@typescript-eslint/project-service@8.59.2(typescript@6.0.3)': + '@typescript-eslint/project-service@8.59.2(supports-color@7.2.0)(typescript@6.0.3)': dependencies: '@typescript-eslint/tsconfig-utils': 8.59.2(typescript@6.0.3) '@typescript-eslint/types': 8.59.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -7446,13 +7474,13 @@ snapshots: dependencies: typescript: 6.0.3 - '@typescript-eslint/type-utils@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/type-utils@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3)': dependencies: '@typescript-eslint/types': 8.59.2 - '@typescript-eslint/typescript-estree': 8.59.2(typescript@6.0.3) - '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - debug: 4.4.3 - eslint: 9.39.4(jiti@2.7.0) + '@typescript-eslint/typescript-estree': 8.59.2(supports-color@7.2.0)(typescript@6.0.3) + '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + debug: 4.4.3(supports-color@7.2.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) ts-api-utils: 2.5.0(typescript@6.0.3) typescript: 6.0.3 transitivePeerDependencies: @@ -7460,13 +7488,13 @@ snapshots: '@typescript-eslint/types@8.59.2': {} - '@typescript-eslint/typescript-estree@8.59.2(typescript@6.0.3)': + '@typescript-eslint/typescript-estree@8.59.2(supports-color@7.2.0)(typescript@6.0.3)': dependencies: - '@typescript-eslint/project-service': 8.59.2(typescript@6.0.3) + '@typescript-eslint/project-service': 8.59.2(supports-color@7.2.0)(typescript@6.0.3) '@typescript-eslint/tsconfig-utils': 8.59.2(typescript@6.0.3) '@typescript-eslint/types': 8.59.2 '@typescript-eslint/visitor-keys': 8.59.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) minimatch: 10.2.5 semver: 7.7.4 tinyglobby: 0.2.16 @@ -7475,13 +7503,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3)': + '@typescript-eslint/utils@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3)': dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.7.0)) + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)) '@typescript-eslint/scope-manager': 8.59.2 '@typescript-eslint/types': 8.59.2 - '@typescript-eslint/typescript-estree': 8.59.2(typescript@6.0.3) - eslint: 9.39.4(jiti@2.7.0) + '@typescript-eslint/typescript-estree': 8.59.2(supports-color@7.2.0)(typescript@6.0.3) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -7635,9 +7663,9 @@ snapshots: acorn@8.16.0: {} - agent-base@6.0.2: + agent-base@6.0.2(supports-color@7.2.0): dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -7784,11 +7812,11 @@ snapshots: axe-core@4.11.4: {} - axios@1.18.1: + axios@1.18.1(debug@4.4.3(supports-color@7.2.0))(supports-color@7.2.0): dependencies: - follow-redirects: 1.16.0 + follow-redirects: 1.16.0(debug@4.4.3(supports-color@7.2.0)) form-data: 4.0.6 - https-proxy-agent: 5.0.1 + https-proxy-agent: 5.0.1(supports-color@7.2.0) proxy-from-env: 2.1.0 transitivePeerDependencies: - debug @@ -7816,11 +7844,11 @@ snapshots: bluebird@3.4.7: {} - body-parser@2.2.2: + body-parser@2.2.2(supports-color@7.2.0): dependencies: bytes: 3.1.2 content-type: 1.0.5 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) http-errors: 2.0.1 iconv-lite: 0.7.2 on-finished: 2.4.1 @@ -8021,13 +8049,17 @@ snapshots: es-errors: 1.3.0 is-data-view: 1.0.2 - debug@3.2.7: + debug@3.2.7(supports-color@7.2.0): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 7.2.0 - debug@4.4.3: + debug@4.4.3(supports-color@7.2.0): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 7.2.0 decimal.js@10.6.0: {} @@ -8350,18 +8382,18 @@ snapshots: escape-string-regexp@5.0.0: {} - eslint-config-next@16.2.4(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3): + eslint-config-next@16.2.4(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3): dependencies: '@next/eslint-plugin-next': 16.2.4 - eslint: 9.39.4(jiti@2.7.0) - eslint-import-resolver-node: 0.3.10 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)) - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)) - eslint-plugin-jsx-a11y: 6.10.2(eslint@9.39.4(jiti@2.7.0)) - eslint-plugin-react: 7.37.5(eslint@9.39.4(jiti@2.7.0)) - eslint-plugin-react-hooks: 7.1.1(eslint@9.39.4(jiti@2.7.0)) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) + eslint-import-resolver-node: 0.3.10(supports-color@7.2.0) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) + eslint-plugin-jsx-a11y: 6.10.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)) + eslint-plugin-react: 7.37.5(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)) + eslint-plugin-react-hooks: 7.1.1(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) globals: 16.4.0 - typescript-eslint: 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) + typescript-eslint: 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) optionalDependencies: typescript: 6.0.3 transitivePeerDependencies: @@ -8370,52 +8402,52 @@ snapshots: - eslint-plugin-import-x - supports-color - eslint-import-resolver-node@0.3.10: + eslint-import-resolver-node@0.3.10(supports-color@7.2.0): dependencies: - debug: 3.2.7 + debug: 3.2.7(supports-color@7.2.0) is-core-module: 2.16.2 resolve: 2.0.0-next.6 transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)): + eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0): dependencies: '@nolyfill/is-core-module': 1.0.39 - debug: 4.4.3 - eslint: 9.39.4(jiti@2.7.0) + debug: 4.4.3(supports-color@7.2.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) get-tsconfig: 4.14.0 is-bun-module: 2.0.0 stable-hash: 0.0.5 tinyglobby: 0.2.16 unrs-resolver: 1.11.1 optionalDependencies: - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.1(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-node@0.3.10)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)): + eslint-module-utils@2.12.1(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint-import-resolver-node@0.3.10(supports-color@7.2.0))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0): dependencies: - debug: 3.2.7 + debug: 3.2.7(supports-color@7.2.0) optionalDependencies: - '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - eslint: 9.39.4(jiti@2.7.0) - eslint-import-resolver-node: 0.3.10 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)) + '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) + eslint-import-resolver-node: 0.3.10(supports-color@7.2.0) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) transitivePeerDependencies: - supports-color - eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)): + eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0): dependencies: '@rtsao/scc': 1.1.0 array-includes: 3.1.9 array.prototype.findlastindex: 1.2.6 array.prototype.flat: 1.3.3 array.prototype.flatmap: 1.3.3 - debug: 3.2.7 + debug: 3.2.7(supports-color@7.2.0) doctrine: 2.1.0 - eslint: 9.39.4(jiti@2.7.0) - eslint-import-resolver-node: 0.3.10 - eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint-import-resolver-node@0.3.10)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) + eslint-import-resolver-node: 0.3.10(supports-color@7.2.0) + eslint-module-utils: 2.12.1(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint-import-resolver-node@0.3.10(supports-color@7.2.0))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) hasown: 2.0.3 is-core-module: 2.16.2 is-glob: 4.0.3 @@ -8427,13 +8459,13 @@ snapshots: string.prototype.trimend: 1.0.9 tsconfig-paths: 3.15.0 optionalDependencies: - '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) + '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) transitivePeerDependencies: - eslint-import-resolver-typescript - eslint-import-resolver-webpack - supports-color - eslint-plugin-jsx-a11y@6.10.2(eslint@9.39.4(jiti@2.7.0)): + eslint-plugin-jsx-a11y@6.10.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)): dependencies: aria-query: 5.3.2 array-includes: 3.1.9 @@ -8443,7 +8475,7 @@ snapshots: axobject-query: 4.1.0 damerau-levenshtein: 1.0.8 emoji-regex: 9.2.2 - eslint: 9.39.4(jiti@2.7.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) hasown: 2.0.3 jsx-ast-utils: 3.3.5 language-tags: 1.0.9 @@ -8452,18 +8484,18 @@ snapshots: safe-regex-test: 1.1.0 string.prototype.includes: 2.0.1 - eslint-plugin-react-hooks@7.1.1(eslint@9.39.4(jiti@2.7.0)): + eslint-plugin-react-hooks@7.1.1(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0): dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/parser': 7.29.3 - eslint: 9.39.4(jiti@2.7.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) hermes-parser: 0.25.1 zod: 4.4.3 zod-validation-error: 4.0.2(zod@4.4.3) transitivePeerDependencies: - supports-color - eslint-plugin-react@7.37.5(eslint@9.39.4(jiti@2.7.0)): + eslint-plugin-react@7.37.5(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)): dependencies: array-includes: 3.1.9 array.prototype.findlast: 1.2.5 @@ -8471,7 +8503,7 @@ snapshots: array.prototype.tosorted: 1.1.4 doctrine: 2.1.0 es-iterator-helpers: 1.3.2 - eslint: 9.39.4(jiti@2.7.0) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) estraverse: 5.3.0 hasown: 2.0.3 jsx-ast-utils: 3.3.5 @@ -8496,14 +8528,14 @@ snapshots: eslint-visitor-keys@5.0.1: {} - eslint@9.39.4(jiti@2.7.0): + eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0): dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.7.0)) + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0)) '@eslint-community/regexpp': 4.12.2 - '@eslint/config-array': 0.21.2 + '@eslint/config-array': 0.21.2(supports-color@7.2.0) '@eslint/config-helpers': 0.4.2 '@eslint/core': 0.17.0 - '@eslint/eslintrc': 3.3.5 + '@eslint/eslintrc': 3.3.5(supports-color@7.2.0) '@eslint/js': 9.39.4 '@eslint/plugin-kit': 0.4.1 '@humanfs/node': 0.16.8 @@ -8513,7 +8545,7 @@ snapshots: ajv: 6.15.0 chalk: 4.1.2 cross-spawn: 7.0.6 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) escape-string-regexp: 4.0.0 eslint-scope: 8.4.0 eslint-visitor-keys: 4.2.1 @@ -8600,25 +8632,25 @@ snapshots: expect-type@1.3.0: {} - express-rate-limit@8.5.0(express@5.2.1): + express-rate-limit@8.5.0(express@5.2.1(supports-color@7.2.0)): dependencies: - express: 5.2.1 + express: 5.2.1(supports-color@7.2.0) ip-address: 10.1.0 - express@5.2.1: + express@5.2.1(supports-color@7.2.0): dependencies: accepts: 2.0.0 - body-parser: 2.2.2 + body-parser: 2.2.2(supports-color@7.2.0) content-disposition: 1.1.0 content-type: 1.0.5 cookie: 0.7.2 cookie-signature: 1.2.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) depd: 2.0.0 encodeurl: 2.0.0 escape-html: 1.0.3 etag: 1.8.1 - finalhandler: 2.1.1 + finalhandler: 2.1.1(supports-color@7.2.0) fresh: 2.0.0 http-errors: 2.0.1 merge-descriptors: 2.0.0 @@ -8629,9 +8661,9 @@ snapshots: proxy-addr: 2.0.7 qs: 6.15.1 range-parser: 1.2.1 - router: 2.2.0 - send: 1.2.1 - serve-static: 2.2.1 + router: 2.2.0(supports-color@7.2.0) + send: 1.2.1(supports-color@7.2.0) + serve-static: 2.2.1(supports-color@7.2.0) statuses: 2.0.2 type-is: 2.0.1 vary: 1.1.2 @@ -8709,9 +8741,9 @@ snapshots: dependencies: to-regex-range: 5.0.1 - finalhandler@2.1.1: + finalhandler@2.1.1(supports-color@7.2.0): dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) encodeurl: 2.0.0 escape-html: 1.0.3 on-finished: 2.4.1 @@ -8734,7 +8766,9 @@ snapshots: flatted@3.4.2: {} - follow-redirects@1.16.0: {} + follow-redirects@1.16.0(debug@4.4.3(supports-color@7.2.0)): + optionalDependencies: + debug: 4.4.3(supports-color@7.2.0) for-each@0.3.5: dependencies: @@ -8886,7 +8920,7 @@ snapshots: dependencies: function-bind: 1.1.2 - hast-util-to-jsx-runtime@2.3.6: + hast-util-to-jsx-runtime@2.3.6(supports-color@7.2.0): dependencies: '@types/estree': 1.0.8 '@types/hast': 3.0.4 @@ -8895,9 +8929,9 @@ snapshots: devlop: 1.1.0 estree-util-is-identifier-name: 3.0.0 hast-util-whitespace: 3.0.0 - mdast-util-mdx-expression: 2.0.1 - mdast-util-mdx-jsx: 3.2.0 - mdast-util-mdxjs-esm: 2.0.1 + mdast-util-mdx-expression: 2.0.1(supports-color@7.2.0) + mdast-util-mdx-jsx: 3.2.0(supports-color@7.2.0) + mdast-util-mdxjs-esm: 2.0.1(supports-color@7.2.0) property-information: 7.1.0 space-separated-tokens: 2.0.2 style-to-js: 1.1.21 @@ -8939,17 +8973,17 @@ snapshots: statuses: 2.0.2 toidentifier: 1.0.1 - https-proxy-agent@5.0.1: + https-proxy-agent@5.0.1(supports-color@7.2.0): dependencies: - agent-base: 6.0.2 - debug: 4.4.3 + agent-base: 6.0.2(supports-color@7.2.0) + debug: 4.4.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color - https-proxy-agent@7.0.6: + https-proxy-agent@7.0.6(supports-color@7.2.0): dependencies: agent-base: 7.1.4 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -9398,14 +9432,14 @@ snapshots: unist-util-is: 6.0.1 unist-util-visit-parents: 6.0.2 - mdast-util-from-markdown@2.0.3: + mdast-util-from-markdown@2.0.3(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 '@types/unist': 3.0.3 decode-named-character-reference: 1.3.0 devlop: 1.1.0 mdast-util-to-string: 4.0.0 - micromark: 4.0.2 + micromark: 4.0.2(supports-color@7.2.0) micromark-util-decode-numeric-character-reference: 2.0.2 micromark-util-decode-string: 2.0.1 micromark-util-normalize-identifier: 2.0.1 @@ -9423,67 +9457,67 @@ snapshots: mdast-util-find-and-replace: 3.0.2 micromark-util-character: 2.1.1 - mdast-util-gfm-footnote@2.1.0: + mdast-util-gfm-footnote@2.1.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 micromark-util-normalize-identifier: 2.0.1 transitivePeerDependencies: - supports-color - mdast-util-gfm-strikethrough@2.0.0: + mdast-util-gfm-strikethrough@2.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-gfm-table@2.0.0: + mdast-util-gfm-table@2.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 devlop: 1.1.0 markdown-table: 3.0.4 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-gfm-task-list-item@2.0.0: + mdast-util-gfm-task-list-item@2.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-gfm@3.1.0: + mdast-util-gfm@3.1.0(supports-color@7.2.0): dependencies: - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-gfm-autolink-literal: 2.0.1 - mdast-util-gfm-footnote: 2.1.0 - mdast-util-gfm-strikethrough: 2.0.0 - mdast-util-gfm-table: 2.0.0 - mdast-util-gfm-task-list-item: 2.0.0 + mdast-util-gfm-footnote: 2.1.0(supports-color@7.2.0) + mdast-util-gfm-strikethrough: 2.0.0(supports-color@7.2.0) + mdast-util-gfm-table: 2.0.0(supports-color@7.2.0) + mdast-util-gfm-task-list-item: 2.0.0(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-mdx-expression@2.0.1: + mdast-util-mdx-expression@2.0.1(supports-color@7.2.0): dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color - mdast-util-mdx-jsx@3.2.0: + mdast-util-mdx-jsx@3.2.0(supports-color@7.2.0): dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 @@ -9491,7 +9525,7 @@ snapshots: '@types/unist': 3.0.3 ccount: 2.0.1 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 parse-entities: 4.0.2 stringify-entities: 4.0.4 @@ -9500,13 +9534,13 @@ snapshots: transitivePeerDependencies: - supports-color - mdast-util-mdxjs-esm@2.0.1: + mdast-util-mdxjs-esm@2.0.1(supports-color@7.2.0): dependencies: '@types/estree-jsx': 1.0.5 '@types/hast': 3.0.4 '@types/mdast': 4.0.4 devlop: 1.1.0 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) mdast-util-to-markdown: 2.1.2 transitivePeerDependencies: - supports-color @@ -9727,10 +9761,10 @@ snapshots: micromark-util-types@2.0.2: {} - micromark@4.0.2: + micromark@4.0.2(supports-color@7.2.0): dependencies: '@types/debug': 4.1.13 - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) decode-named-character-reference: 1.3.0 devlop: 1.1.0 micromark-core-commonmark: 2.0.3 @@ -9842,7 +9876,7 @@ snapshots: react: 19.2.4 react-dom: 19.2.4(react@19.2.4) - next@16.2.4(@babel/core@7.29.0)(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + next@16.2.4(@babel/core@7.29.0(supports-color@7.2.0))(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4): dependencies: '@next/env': 16.2.4 '@swc/helpers': 0.5.15 @@ -9851,7 +9885,7 @@ snapshots: postcss: 8.4.31 react: 19.2.4 react-dom: 19.2.4(react@19.2.4) - styled-jsx: 5.1.6(@babel/core@7.29.0)(react@19.2.4) + styled-jsx: 5.1.6(@babel/core@7.29.0(supports-color@7.2.0))(react@19.2.4) optionalDependencies: '@next/swc-darwin-arm64': 16.2.4 '@next/swc-darwin-x64': 16.2.4 @@ -10183,17 +10217,17 @@ snapshots: react-is@17.0.2: {} - react-markdown@10.1.0(@types/react@19.2.14)(react@19.2.4): + react-markdown@10.1.0(@types/react@19.2.14)(react@19.2.4)(supports-color@7.2.0): dependencies: '@types/hast': 3.0.4 '@types/mdast': 4.0.4 '@types/react': 19.2.14 devlop: 1.1.0 - hast-util-to-jsx-runtime: 2.3.6 + hast-util-to-jsx-runtime: 2.3.6(supports-color@7.2.0) html-url-attributes: 3.0.1 mdast-util-to-hast: 13.2.1 react: 19.2.4 - remark-parse: 11.0.0 + remark-parse: 11.0.0(supports-color@7.2.0) remark-rehype: 11.1.2 unified: 11.0.5 unist-util-visit: 5.1.0 @@ -10283,21 +10317,21 @@ snapshots: gopd: 1.2.0 set-function-name: 2.0.2 - remark-gfm@4.0.1: + remark-gfm@4.0.1(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 - mdast-util-gfm: 3.1.0 + mdast-util-gfm: 3.1.0(supports-color@7.2.0) micromark-extension-gfm: 3.0.0 - remark-parse: 11.0.0 + remark-parse: 11.0.0(supports-color@7.2.0) remark-stringify: 11.0.0 unified: 11.0.5 transitivePeerDependencies: - supports-color - remark-parse@11.0.0: + remark-parse@11.0.0(supports-color@7.2.0): dependencies: '@types/mdast': 4.0.4 - mdast-util-from-markdown: 2.0.3 + mdast-util-from-markdown: 2.0.3(supports-color@7.2.0) micromark-util-types: 2.0.2 unified: 11.0.5 transitivePeerDependencies: @@ -10366,9 +10400,9 @@ snapshots: '@rolldown/binding-win32-arm64-msvc': 1.0.0-rc.17 '@rolldown/binding-win32-x64-msvc': 1.0.0-rc.17 - router@2.2.0: + router@2.2.0(supports-color@7.2.0): dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) depd: 2.0.0 is-promise: 4.0.0 parseurl: 1.3.3 @@ -10420,9 +10454,9 @@ snapshots: semver@7.7.4: {} - send@1.2.1: + send@1.2.1(supports-color@7.2.0): dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@7.2.0) encodeurl: 2.0.0 escape-html: 1.0.3 etag: 1.8.1 @@ -10436,12 +10470,12 @@ snapshots: transitivePeerDependencies: - supports-color - serve-static@2.2.1: + serve-static@2.2.1(supports-color@7.2.0): dependencies: encodeurl: 2.0.0 escape-html: 1.0.3 parseurl: 1.3.3 - send: 1.2.1 + send: 1.2.1(supports-color@7.2.0) transitivePeerDependencies: - supports-color @@ -10475,14 +10509,14 @@ snapshots: setprototypeof@1.2.0: {} - shadcn@4.7.0(@types/node@20.19.39)(typescript@6.0.3): + shadcn@4.7.0(@types/node@20.19.39)(supports-color@7.2.0)(typescript@6.0.3): dependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) '@babel/parser': 7.29.3 - '@babel/plugin-transform-typescript': 7.28.6(@babel/core@7.29.0) - '@babel/preset-typescript': 7.28.5(@babel/core@7.29.0) + '@babel/plugin-transform-typescript': 7.28.6(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) + '@babel/preset-typescript': 7.28.5(@babel/core@7.29.0(supports-color@7.2.0))(supports-color@7.2.0) '@dotenvx/dotenvx': 1.65.0 - '@modelcontextprotocol/sdk': 1.29.0(zod@3.25.76) + '@modelcontextprotocol/sdk': 1.30.0(supports-color@7.2.0)(zod@3.25.76) '@types/validate-npm-package-name': 4.0.2 browserslist: 4.28.2 commander: 14.0.3 @@ -10494,7 +10528,7 @@ snapshots: fast-glob: 3.3.3 fs-extra: 11.3.4 fuzzysort: 3.1.0 - https-proxy-agent: 7.0.6 + https-proxy-agent: 7.0.6(supports-color@7.2.0) kleur: 4.1.5 msw: 2.14.3(@types/node@20.19.39)(typescript@6.0.3) node-fetch: 3.3.2 @@ -10725,12 +10759,12 @@ snapshots: dependencies: inline-style-parser: 0.2.7 - styled-jsx@5.1.6(@babel/core@7.29.0)(react@19.2.4): + styled-jsx@5.1.6(@babel/core@7.29.0(supports-color@7.2.0))(react@19.2.4): dependencies: client-only: 0.0.1 react: 19.2.4 optionalDependencies: - '@babel/core': 7.29.0 + '@babel/core': 7.29.0(supports-color@7.2.0) supports-color@7.2.0: dependencies: @@ -10873,13 +10907,13 @@ snapshots: possible-typed-array-names: 1.1.0 reflect.getprototypeof: 1.0.10 - typescript-eslint@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3): + typescript-eslint@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3): dependencies: - '@typescript-eslint/eslint-plugin': 8.59.2(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - '@typescript-eslint/typescript-estree': 8.59.2(typescript@6.0.3) - '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0))(typescript@6.0.3) - eslint: 9.39.4(jiti@2.7.0) + '@typescript-eslint/eslint-plugin': 8.59.2(@typescript-eslint/parser@8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3))(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + '@typescript-eslint/parser': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + '@typescript-eslint/typescript-estree': 8.59.2(supports-color@7.2.0)(typescript@6.0.3) + '@typescript-eslint/utils': 8.59.2(eslint@9.39.4(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) + eslint: 9.39.4(jiti@2.7.0)(supports-color@7.2.0) typescript: 6.0.3 transitivePeerDependencies: - supports-color @@ -11185,6 +11219,10 @@ snapshots: dependencies: zod: 3.25.76 + zod-to-json-schema@3.25.2(zod@4.4.3): + dependencies: + zod: 4.4.3 + zod-validation-error@4.0.2(zod@4.4.3): dependencies: zod: 4.4.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 80ee5bbc..082ea2dc 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,3 +1,11 @@ +allowBuilds: + core-js: set this to true or false + esbuild: set this to true or false + msgpackr-extract: set this to true or false + msw: set this to true or false + sharp: set this to true or false + unrs-resolver: set this to true or false + ignoredBuiltDependencies: - sharp - unrs-resolver diff --git a/src/agent-harness/index.ts b/src/agent-harness/index.ts index a2666bd4..40e843bb 100644 --- a/src/agent-harness/index.ts +++ b/src/agent-harness/index.ts @@ -1,5 +1,6 @@ export * from "./image-highlights" export * from "./ledger" export * from "./knowhere-text" +export * from "./memory-text" export * from "./runtime" export * from "./types" diff --git a/src/agent-harness/knowhere-text.test.ts b/src/agent-harness/knowhere-text.test.ts index e916cad2..af1094e9 100644 --- a/src/agent-harness/knowhere-text.test.ts +++ b/src/agent-harness/knowhere-text.test.ts @@ -1,10 +1,5 @@ import { describe, expect, it } from "vitest" -import type { - KnowledgeGrepResponse, - KnowledgeOutline, - KnowledgeReadResponse, - RetrievalQueryResponse, -} from "@ontos-ai/knowhere-sdk" +import type { RetrievalQueryResponse } from "@ontos-ai/knowhere-sdk" import { createEvidenceLedger } from "./ledger" import { knowhereToolText } from "./knowhere-text" @@ -18,89 +13,34 @@ describe("knowhereToolText", () => { const text = knowhereToolText.formatSearch({ response, retrievalCount: snapshot.retrievalCount, + chunkPickStart: 0, chunks: snapshot.chunks, assets: snapshot.assets, }) expect(text).toContain('') + expect(text).toContain('pick="1"') expect(text).toContain('ref="r1:result:1"') expect(text).toContain('ref="asset:r1:result:1"') + expect(text).toContain("Page one summary.") expect(text).toContain("Call inspectImage") expect(text).toContain("before finalize") + expect(text).not.toContain("") + expect(text).not.toContain("Page one evidence.") expect(text).not.toContain("https://assets.example/page-1.png") }) - it("formats list and outline responses", () => { - const listText = knowhereToolText.formatListDocuments({ - documents: [ - { - documentId: "doc_1", - revisionKey: "job_1", - namespace: "notebook", - sourceFileName: "report.pdf", - title: "Report", - status: "ready", - }, - ], - }) - const outlineText = knowhereToolText.formatOutline(makeOutlineResponse()) - - expect(listText).toContain('') - expect(listText).toContain('documentId="doc_1"') - expect(outlineText).toContain( - '', - ) - expect(outlineText).toContain('sectionPath="Root / Revenue"') - }) - - it("formats full read chunk bodies without truncating large content", () => { - const ledger = createEvidenceLedger() - const largeContent = `BEGIN ${"full chunk body ".repeat(700)} END` - const response = makeReadResponse(largeContent) - const snapshot = ledger.addReadChunksResponse(response) - - const text = knowhereToolText.formatReadChunks({ - response, - chunks: snapshot.chunks, - assets: snapshot.assets, - }) - - expect(text).toContain('') - expect(text).toContain('ref="read1:chunk:1"') - expect(text).toContain(largeContent) - expect(text).not.toContain("...[truncated]") - expect(text).not.toContain("https://assets.example/page-1.png") - }) - - it("formats grep matches with continuation metadata", () => { - const ledger = createEvidenceLedger() - const response = makeGrepResponse() - const snapshot = ledger.addGrepChunksResponse(response) - - const text = knowhereToolText.formatGrepChunks({ - response, - chunks: snapshot.chunks, - assets: snapshot.assets, - }) - - expect(text).toContain('') - expect(text).toContain('truncated="true"') - expect(text).toContain('continuationCursor="cursor_2"') - expect(text).toContain('ref="grep1:match:1"') - expect(text).toContain("matched penalty snippet") - }) - it("formats errors as tagged text", () => { expect( knowhereToolText.formatError({ - operation: "read_chunks", - message: "A documentId is required.", + operation: "search", + message: "Knowhere search failed.", }), ).toBe( [ - '', + '', "", - "A documentId is required.", + "Knowhere search failed.", "", "", ].join("\n"), @@ -144,116 +84,3 @@ function makeSearchResponse(): RetrievalQueryResponse { referencedChunks: [], } } - -function makeReadResponse(content: string): KnowledgeReadResponse { - return { - document: { - localDocumentId: "doc_1", - documentId: "doc_1", - jobId: "job_1", - namespace: "notebook", - sourceFileName: "report.pdf", - chunkCount: 1, - typeCounts: { text: 0, image: 0, table: 0, page: 1 }, - resultDirectoryPath: "parsed-storage:doc_1/job_1", - createdAt: new Date("2026-01-01T00:00:00Z"), - updatedAt: new Date("2026-01-01T00:00:00Z"), - }, - chunks: [ - { - position: 1, - chunkId: "chunk_page_1", - chunkType: "page", - contentSource: "content", - content, - readableContent: content, - sectionPath: "Root / Page 1", - sourceChunkPath: "pages/page-1.md", - filePath: "pages/page-1.png", - assetUrl: "https://assets.example/page-1.png", - pageNumbers: [1], - metadata: { - pageNums: [1], - pageAssets: [ - { - pageNum: 1, - artifactRef: "page_citation_assets/page-1.png", - assetUrl: "https://assets.example/page-1.png", - contentType: "image/png", - }, - ], - }, - }, - ], - page: 1, - pageSize: 1, - totalChunks: 1, - totalPages: 1, - } -} - -function makeOutlineResponse(): KnowledgeOutline { - return { - document: { - localDocumentId: "doc_1", - documentId: "doc_1", - jobId: "job_1", - namespace: "notebook", - sourceFileName: "report.pdf", - chunkCount: 1, - typeCounts: { text: 1, image: 0, table: 0, page: 0 }, - resultDirectoryPath: "parsed-storage:doc_1/job_1", - createdAt: new Date("2026-01-01T00:00:00Z"), - updatedAt: new Date("2026-01-01T00:00:00Z"), - }, - totalChunks: 1, - typeCounts: { text: 1, image: 0, table: 0, page: 0 }, - sections: [ - { - sectionPath: "Root / Revenue", - sectionTitle: "Revenue", - sectionLevel: 2, - summary: "Revenue summary.", - startChunk: 1, - endChunk: 1, - chunkCount: 1, - typeCounts: { text: 1, image: 0, table: 0, page: 0 }, - children: [], - }, - ], - sectionTree: [], - } -} - -function makeGrepResponse(): KnowledgeGrepResponse { - return { - document: { - localDocumentId: "doc_1", - documentId: "doc_1", - jobId: "job_1", - namespace: "notebook", - sourceFileName: "contract.pdf", - chunkCount: 4, - typeCounts: { text: 4, image: 0, table: 0, page: 0 }, - resultDirectoryPath: "parsed-storage:doc_1/job_1", - createdAt: new Date("2026-01-01T00:00:00Z"), - updatedAt: new Date("2026-01-01T00:00:00Z"), - }, - matches: [ - { - position: 3, - chunkId: "chunk_3", - chunkType: "text", - sectionPath: "Root / Penalties", - sourceChunkPath: "chunks/chunk-3.md", - filePath: "contract.pdf", - startOffset: 10, - endOffset: 17, - snippet: "matched penalty snippet", - }, - ], - scannedChunks: 4, - truncated: true, - continuationCursor: "cursor_2", - } -} diff --git a/src/agent-harness/knowhere-text.ts b/src/agent-harness/knowhere-text.ts index f5b19da3..5f55ff78 100644 --- a/src/agent-harness/knowhere-text.ts +++ b/src/agent-harness/knowhere-text.ts @@ -1,15 +1,6 @@ -import type { - KnowledgeGrepResponse, - KnowledgeOutline, - KnowledgeReadResponse, - RetrievalQueryResponse, -} from "@ontos-ai/knowhere-sdk" +import type { RetrievalQueryResponse } from "@ontos-ai/knowhere-sdk" -import type { - EvidenceAsset, - EvidenceChunk, - KnowhereListDocumentsResponse, -} from "./types" +import type { EvidenceAsset, EvidenceChunk } from "./types" type EvidenceDelta = { readonly chunks: readonly EvidenceChunk[] @@ -19,14 +10,7 @@ type EvidenceDelta = { type SearchTextInput = EvidenceDelta & { readonly response: RetrievalQueryResponse readonly retrievalCount: number -} - -type ReadChunksTextInput = EvidenceDelta & { - readonly response: KnowledgeReadResponse -} - -type GrepChunksTextInput = EvidenceDelta & { - readonly response: KnowledgeGrepResponse + readonly chunkPickStart: number } type ErrorTextInput = { @@ -34,12 +18,7 @@ type ErrorTextInput = { readonly message: string } -type KnowhereOperation = - | "search" - | "list_documents" - | "get_document_outline" - | "read_chunks" - | "grep_chunks" +type KnowhereOperation = "search" const assetInstruction = "Notebook returned image/page asset refs. Call inspectImage with the asset refs you will cite before finalize so OCR/visual context and provenance boxes exist. Do not expose raw asset URLs." @@ -56,100 +35,14 @@ export const knowhereToolText = { stopReason: input.response.stopReason ?? undefined, failureReason: input.response.failureReason ?? undefined, }), - formatOptionalTextTag("evidence", input.response.evidenceText), - formatEvidenceChunks(input.chunks), + // Model grounding comes from (results). Do not also inject + // evidenceText — same bodies, no citeable refs, doubles context. + formatEvidenceChunks(input.chunks, input.chunkPickStart), formatEvidenceAssets(input.assets), formatAssetInstruction(input.assets), ]) }, - formatListDocuments(response: KnowhereListDocumentsResponse): string { - return wrapKnowhereBlock("list_documents", [ - formatTag("summary", { documentCount: String(response.documents.length) }), - ...response.documents.map((document, index) => - formatSelfClosingTag("document", { - index: String(index + 1), - documentId: document.documentId, - localDocumentId: document.localDocumentId, - revisionKey: document.revisionKey, - namespace: document.namespace, - sourceFileName: document.sourceFileName, - title: document.title, - status: document.status, - chunkCount: - typeof document.chunkCount === "number" - ? String(document.chunkCount) - : undefined, - }), - ), - ]) - }, - - formatOutline(response: KnowledgeOutline): string { - return wrapKnowhereBlock("get_document_outline", [ - formatTag("document", { - documentId: response.document.documentId, - localDocumentId: response.document.localDocumentId, - revisionKey: response.document.jobId, - sourceFileName: response.document.sourceFileName, - totalChunks: String(response.totalChunks), - truncated: response.truncated === true ? "true" : undefined, - continuationCursor: response.continuationCursor, - }), - ...response.sections.map((section) => formatSection(section, 0)), - ]) - }, - - formatReadChunks(input: ReadChunksTextInput): string { - return wrapKnowhereBlock("read_chunks", [ - formatTag("document", { - documentId: input.response.document.documentId, - localDocumentId: input.response.document.localDocumentId, - revisionKey: input.response.document.jobId, - sourceFileName: input.response.document.sourceFileName, - page: - typeof input.response.page === "number" - ? String(input.response.page) - : undefined, - pageSize: - typeof input.response.pageSize === "number" - ? String(input.response.pageSize) - : undefined, - totalChunks: - typeof input.response.totalChunks === "number" - ? String(input.response.totalChunks) - : undefined, - totalPages: - typeof input.response.totalPages === "number" - ? String(input.response.totalPages) - : undefined, - nextChunk: - typeof input.response.nextChunk === "number" - ? String(input.response.nextChunk) - : undefined, - }), - formatEvidenceChunks(input.chunks), - formatEvidenceAssets(input.assets), - formatAssetInstruction(input.assets), - ]) - }, - - formatGrepChunks(input: GrepChunksTextInput): string { - return wrapKnowhereBlock("grep_chunks", [ - formatTag("document", { - documentId: input.response.document.documentId, - localDocumentId: input.response.document.localDocumentId, - revisionKey: input.response.document.jobId, - sourceFileName: input.response.document.sourceFileName, - matchCount: String(input.response.matches.length), - scannedChunks: String(input.response.scannedChunks), - truncated: input.response.truncated ? "true" : "false", - continuationCursor: input.response.continuationCursor, - }), - formatEvidenceChunks(input.chunks), - ]) - }, - formatError(input: ErrorTextInput): string { return [ formatOpenTag("knowhere", { @@ -173,14 +66,18 @@ function wrapKnowhereBlock( ].join("\n") } -function formatEvidenceChunks(chunks: readonly EvidenceChunk[]): string { +function formatEvidenceChunks( + chunks: readonly EvidenceChunk[], + chunkPickStart: number, +): string { if (chunks.length === 0) return "" return [ "", - ...chunks.map((chunk) => + ...chunks.map((chunk, index) => [ formatOpenTag("chunk", { + pick: String(chunkPickStart + index + 1), ref: chunk.ref, kind: chunk.kind, chunkId: chunk.chunkId, @@ -227,42 +124,6 @@ function formatAssetInstruction(assets: readonly EvidenceAsset[]): string { return formatTextTag("asset_instruction", assetInstruction) } -function formatSection( - section: KnowledgeOutline["sections"][number], - depth: number, -): string { - return [ - formatOpenTag("section", { - depth: String(depth), - sectionPath: section.sectionPath, - sectionTitle: section.sectionTitle, - sectionLevel: String(section.sectionLevel), - startChunk: - typeof section.startChunk === "number" - ? String(section.startChunk) - : undefined, - endChunk: - typeof section.endChunk === "number" - ? String(section.endChunk) - : undefined, - chunkCount: String(section.chunkCount), - }), - formatOptionalTextTag("summary", section.summary), - ...section.children.map((child) => formatSection(child, depth + 1)), - "", - ] - .filter((part) => part.trim().length > 0) - .join("\n") -} - -function formatOptionalTextTag( - tagName: string, - value: string | null | undefined, -): string { - const trimmedValue = value?.trim() - return trimmedValue ? formatTextTag(tagName, trimmedValue) : "" -} - function formatTextTag(tagName: string, value: string): string { return [`<${tagName}>`, value, ``].join("\n") } diff --git a/src/agent-harness/ledger.test.ts b/src/agent-harness/ledger.test.ts index 3af12759..6d3bdb32 100644 --- a/src/agent-harness/ledger.test.ts +++ b/src/agent-harness/ledger.test.ts @@ -1,9 +1,5 @@ import { describe, expect, it } from "vitest" -import type { - KnowledgeGrepResponse, - KnowledgeReadResponse, - RetrievalQueryResponse, -} from "@ontos-ai/knowhere-sdk" +import type { RetrievalQueryResponse } from "@ontos-ai/knowhere-sdk" import { createEvidenceLedger } from "./ledger" @@ -117,118 +113,80 @@ describe("createEvidenceLedger", () => { ) }) - it("adds read chunk refs and page image assets", () => { + it("does not throw when a result is missing chunkType", () => { + // Knowhere's chunkType is declared as a required string in the SDK + // type, but real retrieval results can omit it at runtime, same as the + // referencedChunks case above. Unlike referencedChunks (which carry no + // real content), results carry real content, so the chunk must still be + // kept in the ledger -- only asset-type detection should be guarded. const ledger = createEvidenceLedger() - const snapshot = ledger.addReadChunksResponse(makeReadResponse()) - - expect(snapshot.chunks).toEqual([ - expect.objectContaining({ - ref: "read1:chunk:1", - kind: "read_chunk", - content: "Full page content.", - contentPreview: "Full page content.", - assetRef: "asset:read1:chunk:1", - }), - ]) - expect(snapshot.assets).toEqual([ - expect.objectContaining({ - ref: "asset:read1:chunk:1", - chunkRef: "read1:chunk:1", - type: "image", - sourcePath: "page_citation_assets/page-1.png", - }), - ]) - }) - - it("adds grep match refs", () => { - const ledger = createEvidenceLedger() - - const snapshot = ledger.addGrepChunksResponse(makeGrepResponse()) - - expect(snapshot.chunks).toEqual([ - expect.objectContaining({ - ref: "grep1:match:1", - kind: "grep_match", - chunkId: "chunk_3", - content: "matched penalty snippet", - source: expect.objectContaining({ - documentId: "doc_contract", - sourceFileName: "contract.pdf", - sectionPath: "Root / Penalties", - }), - }), - ]) - }) - - it("copies page metadata onto grep matches from the same chunk id", () => { - const ledger = createEvidenceLedger() - ledger.addReadChunksResponse(makeReadResponse()) - - const snapshot = ledger.addGrepChunksResponse({ - ...makeGrepResponse(), - matches: [ + const snapshot = ledger.addRetrievalResponse({ + namespace: "default", + query: "hypertension target", + routerUsed: "agent_explore", + answerText: "", + evidenceText: "[E1] some evidence", + stopReason: "finished", + failureReason: null, + results: [ { - position: 1, - chunkId: "chunk_page_1", - chunkType: "page", - sectionPath: "Root / Page 1", - sourceChunkPath: "pages/page-1.md", - startOffset: 0, - endOffset: 12, - snippet: "Full page snip", + content: "Target BP is <130/80 mmHg.", + chunkType: undefined as unknown as string, + score: 0.9, + assetUrl: "https://assets.example/images/chart.png", + source: { + documentId: "doc_1", + sourceFileName: "guideline.pdf", + sectionPath: "BP targets", + }, }, ], + referencedChunks: [], }) - expect(snapshot.chunks[1]).toEqual( - expect.objectContaining({ - ref: "grep1:match:1", - kind: "grep_match", - chunkId: "chunk_page_1", - metadata: expect.objectContaining({ - pageNums: [1], - position: 1, - startOffset: 0, - endOffset: 12, - }), - }), - ) + expect(snapshot.chunks.map((chunk) => chunk.ref)).toEqual(["r1:result:1"]) + expect(snapshot.chunks[0]?.content).toBe("Target BP is <130/80 mmHg.") + // chunkType is unknown, but the assetUrl itself has an image extension, + // so asset detection still recognizes it as an image via the URL check. + expect(snapshot.assets).toEqual([ + expect.objectContaining({ ref: "asset:r1:result:1", type: "image" }), + ]) }) - it("copies pageNumbers from the grep match when the SDK provides them", () => { + it("skips agent_explore referencedChunks that lack chunkType instead of throwing", () => { const ledger = createEvidenceLedger() - const snapshot = ledger.addGrepChunksResponse({ - ...makeGrepResponse(), - matches: [ + const snapshot = ledger.addRetrievalResponse({ + namespace: "default", + query: "hypertension CAD blood pressure target", + routerUsed: "agent_explore", + answerText: "", + evidenceText: "[E1] some evidence", + stopReason: "finished", + failureReason: null, + results: [ { - position: 1, - chunkId: "chunk_page_4", - chunkType: "page", - sectionPath: "FINANCIAL SUMMARY", - sourceChunkPath: "pages/page-4.md", - startOffset: 0, - endOffset: 12, - snippet: "automotive revenues", - pageNumbers: [4], + content: "Target BP is <130/80 mmHg.", + chunkType: "text", + score: 0.9, + source: { + documentId: "doc_1", + sourceFileName: "guideline.pdf", + sectionPath: "BP targets", + }, }, ], + // Real agent_explore responses can return referencedChunks entries + // that only carry a summary id, with no chunkType/chunkId/documentId + // even though the SDK type declares those as required strings. + referencedChunks: [ + { summary: "8da0776b-c52b-5602-8579-25c421706f5f" }, + ] as unknown as RetrievalQueryResponse["referencedChunks"], }) - expect(snapshot.chunks[0]).toEqual( - expect.objectContaining({ - ref: "grep1:match:1", - kind: "grep_match", - chunkId: "chunk_page_4", - metadata: expect.objectContaining({ - pageNums: [4], - position: 1, - startOffset: 0, - endOffset: 12, - }), - }), - ) + expect(snapshot.chunks.map((chunk) => chunk.ref)).toEqual(["r1:result:1"]) + expect(snapshot.chunks[0]?.content).toBe("Target BP is <130/80 mmHg.") }) }) @@ -302,77 +260,3 @@ function makePageAssetUrlRetrievalResponse(): RetrievalQueryResponse { ], } } - -function makeReadResponse(): KnowledgeReadResponse { - return { - document: { - localDocumentId: "doc_contract", - documentId: "doc_contract", - jobId: "job_contract", - namespace: "notebook", - sourceFileName: "contract.pdf", - chunkCount: 1, - typeCounts: { text: 0, image: 0, table: 0, page: 1 }, - resultDirectoryPath: "parsed-storage:doc_contract/job_contract", - createdAt: new Date("2026-01-01T00:00:00Z"), - updatedAt: new Date("2026-01-01T00:00:00Z"), - }, - chunks: [ - { - position: 1, - chunkId: "chunk_page_1", - chunkType: "page", - content: "Full page content.", - readableContent: "Full page content.", - sectionPath: "Root / Page 1", - sourceChunkPath: "pages/page-1.md", - filePath: "pages/page-1.png", - assetUrl: "https://assets.example/page-1.png", - pageNumbers: [1], - metadata: { - pageNums: [1], - pageAssets: [ - { - pageNum: 1, - artifactRef: "page_citation_assets/page-1.png", - assetUrl: "https://assets.example/page-1.png", - contentType: "image/png", - }, - ], - }, - }, - ], - } -} - -function makeGrepResponse(): KnowledgeGrepResponse { - return { - document: { - localDocumentId: "doc_contract", - documentId: "doc_contract", - jobId: "job_contract", - namespace: "notebook", - sourceFileName: "contract.pdf", - chunkCount: 4, - typeCounts: { text: 4, image: 0, table: 0, page: 0 }, - resultDirectoryPath: "parsed-storage:doc_contract/job_contract", - createdAt: new Date("2026-01-01T00:00:00Z"), - updatedAt: new Date("2026-01-01T00:00:00Z"), - }, - matches: [ - { - position: 3, - chunkId: "chunk_3", - chunkType: "text", - sectionPath: "Root / Penalties", - sourceChunkPath: "chunks/chunk-3.md", - filePath: "contract.pdf", - startOffset: 10, - endOffset: 17, - snippet: "matched penalty snippet", - }, - ], - scannedChunks: 4, - truncated: false, - } -} diff --git a/src/agent-harness/ledger.ts b/src/agent-harness/ledger.ts index 42f31207..9c80448d 100644 --- a/src/agent-harness/ledger.ts +++ b/src/agent-harness/ledger.ts @@ -1,8 +1,4 @@ import type { - KnowledgeGrepMatch, - KnowledgeGrepResponse, - KnowledgeReadChunk, - KnowledgeReadResponse, RetrievalQueryResponse, RetrievalResult, } from "@ontos-ai/knowhere-sdk" @@ -18,8 +14,6 @@ const imageExtensions = [".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg"] as co type MutableLedger = { retrievalCount: number - readCount: number - grepCount: number chunks: EvidenceChunk[] assets: EvidenceAsset[] evidenceText: string[] @@ -47,8 +41,6 @@ export type EvidenceLedger = ReturnType export function createEvidenceLedger() { const ledger: MutableLedger = { retrievalCount: 0, - readCount: 0, - grepCount: 0, chunks: [], assets: [], evidenceText: [], @@ -84,6 +76,15 @@ export function createEvidenceLedger() { }) response.referencedChunks.forEach((chunk, index) => { + // Knowhere's agent_explore router returns referencedChunks entries + // that may carry only a summary id with no chunkType/content + // (despite the SDK type declaring chunkType as required). Skip + // entries missing a usable chunkType: they have no real content + // (content is always "" here) and chunkType is required downstream + // (asset-type detection calls chunkType.toLowerCase()). + if (typeof chunk.chunkType !== "string" || chunk.chunkType.trim().length === 0) { + return + } const content = "" addChunk({ ledger, @@ -112,38 +113,6 @@ export function createEvidenceLedger() { return snapshot(ledger) }, - addReadChunksResponse(response: KnowledgeReadResponse): EvidenceLedgerSnapshot { - ledger.readCount += 1 - const readIndex = ledger.readCount - - response.chunks.forEach((chunk, index) => { - addChunkFromReadChunk({ - ledger, - response, - chunk, - ref: `read${readIndex}:chunk:${index + 1}`, - }) - }) - - return snapshot(ledger) - }, - - addGrepChunksResponse(response: KnowledgeGrepResponse): EvidenceLedgerSnapshot { - ledger.grepCount += 1 - const grepIndex = ledger.grepCount - - response.matches.forEach((match, index) => { - addChunkFromGrepMatch({ - ledger, - response, - match, - ref: `grep${grepIndex}:match:${index + 1}`, - }) - }) - - return snapshot(ledger) - }, - read(ref: string, offset = 0, limit = 4_000) { const chunk = ledger.chunks.find((candidate) => candidate.ref === ref) if (!chunk) { @@ -177,13 +146,6 @@ export function createEvidenceLedger() { return ledger.chunks.length > 0 || ledger.evidenceText.length > 0 }, - hasRef(ref: string): boolean { - return ( - ledger.chunks.some((chunk) => chunk.ref === ref) || - ledger.assets.some((asset) => asset.ref === ref) - ) - }, - snapshot(): EvidenceLedgerSnapshot { return snapshot(ledger) }, @@ -219,100 +181,6 @@ function addChunkFromResult(input: { }) } -function addChunkFromReadChunk(input: { - readonly ledger: MutableLedger - readonly response: KnowledgeReadResponse - readonly chunk: KnowledgeReadChunk - readonly ref: string -}): void { - addChunk({ - ledger: input.ledger, - chunk: { - ref: input.ref, - kind: "read_chunk", - chunkId: input.chunk.chunkId, - content: input.chunk.content, - contentPreview: buildContentPreview(input.chunk.content), - chunkType: input.chunk.chunkType, - score: null, - sourceChunkPath: input.chunk.sourceChunkPath, - filePath: input.chunk.filePath, - metadata: input.chunk.metadata, - source: { - documentId: input.response.document.documentId, - sourceFileName: input.response.document.sourceFileName, - sectionPath: input.chunk.sectionPath, - }, - revisionKey: input.response.document.jobId, - ...(input.chunk.assetUrl ? { assetUrl: input.chunk.assetUrl } : {}), - }, - }) -} - -function addChunkFromGrepMatch(input: { - readonly ledger: MutableLedger - readonly response: KnowledgeGrepResponse - readonly match: KnowledgeGrepMatch - readonly ref: string -}): void { - const donor = input.ledger.chunks.find( - (chunk) => - chunk.chunkId === input.match.chunkId && hasPageMetadata(chunk.metadata), - ) - const pageNums = - input.match.pageNumbers && input.match.pageNumbers.length > 0 - ? [...input.match.pageNumbers] - : undefined - - addChunk({ - ledger: input.ledger, - chunk: { - ref: input.ref, - kind: "grep_match", - chunkId: input.match.chunkId, - content: input.match.snippet, - contentPreview: buildContentPreview(input.match.snippet), - chunkType: input.match.chunkType, - score: null, - sourceChunkPath: input.match.sourceChunkPath, - filePath: input.match.filePath, - metadata: { - ...(donor?.metadata ?? {}), - ...(pageNums ? { pageNums } : {}), - position: input.match.position, - startOffset: input.match.startOffset, - endOffset: input.match.endOffset, - }, - source: { - documentId: input.response.document.documentId, - sourceFileName: input.response.document.sourceFileName, - sectionPath: input.match.sectionPath, - }, - revisionKey: input.response.document.jobId, - }, - }) -} - -function hasPageMetadata( - metadata: Readonly> | undefined, -): boolean { - if (!metadata) return false - const values = [ - metadata.pageNums, - metadata.page_nums, - metadata.pageNum, - metadata.page_num, - metadata.pageAssets, - metadata.page_assets, - ] - return values.some((value) => { - if (Array.isArray(value)) return value.length > 0 - if (typeof value === "number") return Number.isSafeInteger(value) && value > 0 - if (typeof value === "string") return value.trim().length > 0 - return false - }) -} - function addChunk(input: { readonly ledger: MutableLedger readonly chunk: Omit @@ -347,8 +215,16 @@ function buildContentPreview(content: string): string { return `${normalized.slice(0, contentPreviewLimit)}...` } +// Knowhere's chunkType is declared as a required string in the SDK type, +// but real API responses (seen on referencedChunks; results are the same +// contract) can omit it. Normalize defensively instead of calling +// .toLowerCase() on a value that may be undefined at runtime. +function normalizeChunkType(chunkType: string): string { + return typeof chunkType === "string" ? chunkType.toLowerCase() : "" +} + function isRenderableAsset(chunkType: string, assetUrl: string): boolean { - const normalizedChunkType = chunkType.toLowerCase() + const normalizedChunkType = normalizeChunkType(chunkType) return ( normalizedChunkType === "image" || normalizedChunkType === "table" || @@ -379,7 +255,7 @@ function getEvidenceAssetCandidate( function getPageCitationAssetCandidate( chunk: Omit, ): EvidenceAssetCandidate | null { - if (chunk.chunkType.toLowerCase() !== "page") return null + if (normalizeChunkType(chunk.chunkType) !== "page") return null const candidates = [ ...parsePageCitationAssetCandidates(chunk.metadata?.pageAssets), @@ -409,7 +285,7 @@ function getPageCitationAssetCandidate( } function getAssetType(chunkType: string, assetUrl: string): "image" | "table" { - return chunkType.toLowerCase() === "table" && !isImageAssetUrl(assetUrl) + return normalizeChunkType(chunkType) === "table" && !isImageAssetUrl(assetUrl) ? "table" : "image" } diff --git a/src/agent-harness/memory-text.test.ts b/src/agent-harness/memory-text.test.ts new file mode 100644 index 00000000..aa1699de --- /dev/null +++ b/src/agent-harness/memory-text.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest" + +import { memoryToolText } from "./memory-text" + +describe("memoryToolText", () => { + it("formats search results with memory refs and stored summaries", () => { + const text = memoryToolText.formatSearch({ + query: "毛利率", + items: [ + { + ref: "mem:1", + itemId: "item_1", + kind: "stance", + abstractL0: "关注毛利率下滑", + overviewL1: "用户把毛利率当作核心观察指标。", + }, + ], + }) + + expect(text).toContain('') + expect(text).toContain('query="毛利率"') + expect(text).toContain('ref="mem:1"') + expect(text).toContain('itemId="item_1"') + expect(text).toContain('kind="stance"') + expect(text).toContain("关注毛利率下滑") + expect(text).toContain("用户把毛利率当作核心观察指标。") + }) + + it("formats an empty search without inventing items", () => { + const text = memoryToolText.formatSearch({ + query: "unknown", + items: [], + }) + + expect(text).toContain('resultCount="0"') + expect(text).not.toContain("", + ].join("\n") + }, +} as const + +function formatMemoryItems(response: MemorySearchResponse): string { + if (response.items.length === 0) return "" + + return [ + "", + ...response.items.map((item) => + [ + formatOpenTag("item", { + ref: item.ref, + itemId: item.itemId, + kind: item.kind, + }), + formatTextTag("abstract_l0", item.abstractL0), + formatTextTag("overview_l1", item.overviewL1), + "", + ].join("\n"), + ), + "", + ].join("\n") +} + +function wrapMemoryBlock( + operation: MemoryOperation, + parts: readonly string[], +): string { + return [ + formatOpenTag("memory", { operation, status: "ok" }), + ...parts.filter((part) => part.trim().length > 0), + "", + ].join("\n") +} + +function formatTextTag(tagName: string, value: string): string { + return [`<${tagName}>`, value, ``].join("\n") +} + +function formatTag( + tagName: string, + attrs: Readonly>, +): string { + return `${formatOpenTag(tagName, attrs)}` +} + +function formatOpenTag( + tagName: string, + attrs: Readonly>, +): string { + const serializedAttrs = Object.entries(attrs) + .filter((entry): entry is [string, string] => typeof entry[1] === "string") + .map(([key, value]) => `${key}="${escapeAttribute(value)}"`) + .join(" ") + return serializedAttrs ? `<${tagName} ${serializedAttrs}>` : `<${tagName}>` +} + +function escapeAttribute(value: string): string { + return value + .replaceAll("&", "&") + .replaceAll('"', """) + .replaceAll("<", "<") + .replaceAll(">", ">") +} diff --git a/src/agent-harness/runtime.test.ts b/src/agent-harness/runtime.test.ts index 0a5ca193..d36ca1be 100644 --- a/src/agent-harness/runtime.test.ts +++ b/src/agent-harness/runtime.test.ts @@ -16,10 +16,31 @@ import type { ImageInspectionRequest, IntentFrame, KnowhereToolRuntime, + MemoryToolRuntime, OutputManifest, } from "./types" describe("agent harness runtime", () => { + it("tells the agent to search fluid memory first and not treat every question as document retrieval", () => { + const prompt = buildHarnessSystemPrompt(makeTurnInput()) + + expect(prompt).toContain("Call memory_search first") + expect(prompt).toContain( + "Call knowhere_search only when memory is insufficient and groundingPolicy requires citing source documents", + ) + expect(prompt).toContain( + "call knowhere_search again with a refined query", + ) + expect(prompt).toContain("Refine knowhere_search at most twice") + expect(prompt).not.toContain("knowhere_list_documents") + expect(prompt).not.toContain("knowhere_get_document_outline") + expect(prompt).not.toContain("knowhere_read_chunks") + expect(prompt).not.toContain("knowhere_grep_chunks") + expect(prompt).toContain( + "Do not treat every question as a document-retrieval task", + ) + }) + it("keeps KNOWHERE as an evidence provider instead of exposing internal navigation", () => { const prompt = buildHarnessSystemPrompt(makeTurnInput()) @@ -29,16 +50,14 @@ describe("agent harness runtime", () => { expect(prompt).not.toContain("navigation action") }) - it("tells the agent citation metadata is optional and ledger-resolved", () => { + it("tells the agent to cite by search-chunk pick numbers", () => { const prompt = buildHarnessSystemPrompt(makeTurnInput()) - expect(prompt).toContain( - "Citation label and source metadata are optional", - ) - expect(prompt).toContain( - "Notebook resolves citation metadata from evidence refs when possible", - ) - expect(prompt).not.toContain("must match the selected evidence ref exactly") + expect(prompt).toContain("citations is a list of { pick }") + expect(prompt).toContain("Notebook writes the citation list from those picks") + expect(prompt).toContain("Do not pass documentId or evidence refs as citations") + expect(prompt).not.toContain("Citation refs must be evidence refs") + expect(prompt).not.toContain("Citation label and source metadata are optional") }) it("tells the agent to emit [[cite:n]] markers instead of title/pN or [1]", () => { @@ -72,6 +91,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(query), recentTurns: [], }) @@ -84,6 +104,7 @@ describe("agent harness runtime", () => { }) expect(result).toContain('') + expect(result).toContain('pick="1"') expect(result).toContain('ref="r1:result:1"') expect(result).toContain('ref="asset:r1:result:1"') expect(query).toHaveBeenCalledWith({ @@ -103,6 +124,19 @@ describe("agent harness runtime", () => { ]) }) + it.each([ + { includeDocumentIds: ["doc_1"], excludeDocumentIds: ["doc_2"] }, + { includeDocumentIds: [], excludeDocumentIds: [] }, + ])("preserves document scopes in the actual search tool: %j", async (scope) => { + const search = vi.fn().mockResolvedValue(makeRetrievalResponse()) + const tools = createHarnessTools({ + state: {}, ledger: createEvidenceLedger(), recentTurns: [], + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(search), + }) + await executeTool(tools.knowhere_search, { query: "question", ...scope }) + expect(search).toHaveBeenCalledWith(expect.objectContaining(scope)) + }) + it("returns only newly searched evidence in each knowhere_search tool result", async () => { const query = vi .fn() @@ -147,6 +181,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(query), recentTurns: [], }) @@ -154,9 +189,12 @@ describe("agent harness runtime", () => { const firstResult = await executeTool(tools.knowhere_search, { query: "first" }) const secondResult = await executeTool(tools.knowhere_search, { query: "second" }) + expect(firstResult).toContain('pick="1"') expect(firstResult).toContain('ref="r1:result:1"') expect(secondResult).toContain('retrievalCount="2"') - expect(secondResult).toContain("Second evidence") + expect(secondResult).toContain("Second retrieval evidence.") + expect(secondResult).not.toContain("") + expect(secondResult).toContain('pick="2"') expect(secondResult).toContain('ref="r2:result:1"') expect(JSON.stringify(secondResult)).not.toContain("r1:result:1") expect(ledger.snapshot().chunks.map((chunk) => chunk.ref)).toEqual([ @@ -170,6 +208,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -202,6 +241,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -243,6 +283,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -273,6 +314,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -329,6 +371,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -434,6 +477,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -449,7 +493,8 @@ describe("agent harness runtime", () => { expect( await executeTool(tools.finalize, { text: "Revenue was $24.9B [[cite:1]] [[cite:2]].", - citations: [{ ref: "r1:result:1" }, { ref: "r1:result:2" }], + citations: [{ pick: 1 }, { pick: 2 }], + memoryCitations: [], artifacts: [], unresolved: [], }), @@ -522,6 +567,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages, recentTurns: [], @@ -546,6 +592,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages: vi.fn().mockResolvedValue({ analysis: "", @@ -573,7 +620,8 @@ describe("agent harness runtime", () => { const finalize = await executeTool(tools.finalize, { text: "The amount is 5000 yuan [[cite:1]].", - citations: [{ ref: "r1:referenced:1" }], + citations: [{ pick: 1 }], + memoryCitations: [], artifacts: [], unresolved: [], }) @@ -584,6 +632,98 @@ describe("agent harness runtime", () => { expect(state.finalized).not.toBe(true) }) + it("writes citation refs from ledger picks and rejects picks outside the ledger", async () => { + const ledger = createEvidenceLedger() + ledger.addRetrievalResponse(makeRetrievalResponse()) + const state: { + finalized?: boolean + finalizedManifest?: OutputManifest + } = {} + const tools = createHarnessTools({ + state, + ledger, + memoryTools: makeMemoryTools(), + knowhereTools: makeKnowhereTools(), + recentTurns: [], + }) + + const rejected = await executeTool(tools.finalize, { + text: "Target is <130/80 mmHg [[cite:1]].", + citations: [{ pick: 99 }], + memoryCitations: [], + artifacts: [], + unresolved: [], + }) + + expect(rejected).toMatchObject({ + ok: false, + unknownPicks: [99], + }) + expect(String((rejected as { message: string }).message)).toContain( + "Available picks: 1-1", + ) + expect(state.finalized).not.toBe(true) + + const accepted = await executeTool(tools.finalize, { + text: "Target is <130/80 mmHg [[cite:1]].", + citations: [{ pick: 1 }], + memoryCitations: [], + artifacts: [], + unresolved: [], + }) + expect(accepted).toMatchObject({ + ok: true, + citations: [{ ref: "r1:result:1" }], + }) + expect(state.finalized).toBe(true) + expect(state.finalizedManifest?.citations).toEqual([{ ref: "r1:result:1" }]) + }) + + it("maps finalize picks across successive searches", async () => { + const ledger = createEvidenceLedger() + ledger.addRetrievalResponse(makeRetrievalResponse()) + ledger.addRetrievalResponse({ + ...makeRetrievalResponse(), + query: "second query", + results: [ + { + content: "Second retrieval evidence.", + chunkType: "text", + score: 0.8, + source: { + documentId: "doc_2", + sourceFileName: "second.pdf", + sectionPath: "Second", + }, + }, + ], + }) + const state: { + finalizedManifest?: OutputManifest + } = {} + const tools = createHarnessTools({ + state, + ledger, + memoryTools: makeMemoryTools(), + knowhereTools: makeKnowhereTools(), + recentTurns: [], + }) + + const accepted = await executeTool(tools.finalize, { + text: "Second source [[cite:1]].", + citations: [{ pick: 2 }], + memoryCitations: [], + artifacts: [], + unresolved: [], + }) + + expect(accepted).toMatchObject({ + ok: true, + citations: [{ ref: "r2:result:1" }], + }) + expect(state.finalizedManifest?.citations).toEqual([{ ref: "r2:result:1" }]) + }) + it("accepts finalize output without planning-tool gating", async () => { const state: { finalizedManifest?: OutputManifest @@ -592,6 +732,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), recentTurns: [], }) @@ -599,6 +740,7 @@ describe("agent harness runtime", () => { const manifest = { text: "Answer.", citations: [], + memoryCitations: [], artifacts: [], unresolved: [], } @@ -611,6 +753,58 @@ describe("agent harness runtime", () => { expect(state.finalized).toBe(true) }) + it("returns memory refs from memory_search and stores memoryCitations on finalize", async () => { + const search = vi.fn().mockResolvedValue({ + query: "毛利率", + items: [ + { + ref: "mem:1", + itemId: "item_1", + kind: "stance", + abstractL0: "关注毛利率下滑", + overviewL1: "用户把毛利率当作核心观察指标。", + }, + ], + }) + const state: { + finalizedManifest?: OutputManifest + finalized?: boolean + } = {} + const tools = createHarnessTools({ + state, + ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(search), + knowhereTools: makeKnowhereTools(), + recentTurns: [], + }) + + const searchText = await executeTool(tools.memory_search, { + query: "毛利率", + }) + expect(searchText).toContain('') + expect(searchText).toContain('ref="mem:1"') + expect(searchText).toContain('itemId="item_1"') + expect(search).toHaveBeenCalledWith({ + query: "毛利率", + kinds: undefined, + }) + + const manifest = { + text: "按已有记忆，毛利率是核心观察指标。", + citations: [], + memoryCitations: [ + { ref: "mem:1", itemId: "item_1", kind: "stance" as const }, + ], + artifacts: [], + unresolved: [], + } + expect(await executeTool(tools.finalize, manifest)).toMatchObject({ + ok: true, + memoryCitations: manifest.memoryCitations, + }) + expect(state.finalizedManifest).toEqual(manifest) + }) + it("rejects finalize of cited page images until inspectImage has run", async () => { const ledger = createEvidenceLedger() ledger.addRetrievalResponse(makePageCitationRetrievalResponse()) @@ -622,6 +816,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages: vi.fn().mockResolvedValue({ analysis: "The clause shows 5000 yuan per occurrence.", @@ -638,7 +833,8 @@ describe("agent harness runtime", () => { }) const manifest = { text: "The contractor pays 5000 yuan per occurrence [[cite:1]].", - citations: [{ ref: "r1:referenced:1" }], + citations: [{ pick: 1 }], + memoryCitations: [], artifacts: [], unresolved: [], } @@ -691,6 +887,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state: {}, ledger, + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), inspectImages: vi.fn(), recentTurns: [], @@ -698,7 +895,8 @@ describe("agent harness runtime", () => { const result = await executeTool(tools.finalize, { text: "The contractor pays 5000 yuan [[cite:1]].", - citations: [{ ref: "r1:result:1" }], + citations: [{ pick: 1 }], + memoryCitations: [], artifacts: [], unresolved: [], }) @@ -724,6 +922,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), recentTurns: [ { @@ -756,6 +955,7 @@ describe("agent harness runtime", () => { const tools = createHarnessTools({ state, ledger: createEvidenceLedger(), + memoryTools: makeMemoryTools(), knowhereTools: makeKnowhereTools(), recentTurns: [ { @@ -880,7 +1080,7 @@ describe("agent harness runtime", () => { ]) }) - it("keeps normal steps unconstrained before the finalization step", () => { + it("keeps retrieval tools closed until declareIntent allows them", () => { const result = prepareHarnessStep({ stepNumber: 11, messages: [ @@ -891,14 +1091,121 @@ describe("agent harness runtime", () => { ], }) - expect(result).toEqual({ + expect(result.activeTools).toEqual([ + "declareIntent", + "setContextPolicy", + "inspectImage", + "readPriorTurn", + ]) + expect(result.activeTools).not.toContain("finalize") + expect(result.activeTools).not.toContain("memory_search") + expect(result.activeTools).not.toContain("knowhere_search") + }) + + it("opens only memory_search after intent says retrieval may be needed", () => { + const result = prepareHarnessStep({ + stepNumber: 3, + intent: { + task: "answer", + dependsOnPreviousTurn: false, + retrievalNeeded: "maybe", + targetModalities: ["text"], + constraints: {}, + groundingPolicy: "can_use_context", + }, + messages: [], + }) + + expect(result.activeTools).toContain("memory_search") + expect(result.activeTools).toContain("finalize") + expect(result.activeTools).not.toContain("knowhere_search") + }) + + it("keeps Knowhere tools closed for no_retrieval", () => { + const result = prepareHarnessStep({ + stepNumber: 4, + intent: { + task: "answer", + dependsOnPreviousTurn: false, + retrievalNeeded: "no", + targetModalities: ["text"], + constraints: {}, + groundingPolicy: "no_retrieval", + }, + messages: [], + }) + + expect(result.activeTools).toContain("finalize") + expect(result.activeTools).not.toContain("memory_search") + expect(result.activeTools).not.toContain("knowhere_search") + }) + + it("forces a tool call on ordinary steps so the model cannot skip finalize with bare text", () => { + const result = prepareHarnessStep({ + stepNumber: 3, + messages: [], + }) + + expect(result.toolChoice).toBe("required") + expect(result.activeTools).not.toContain("finalize") + }) + + it("blocks finalize on the first step so a document question cannot skip search", () => { + const result = prepareHarnessStep({ + stepNumber: 1, messages: [ { role: "user", - content: "Find the penalty amount.", + content: "高血压合并冠心病，血压目标一般怎么定？", }, ], }) + + expect(result.activeTools).not.toContain("finalize") + expect(result.toolChoice).toBe("required") + }) + + it("opens memory_search and knowhere_search together as peers when sources are required", () => { + const result = prepareHarnessStep({ + stepNumber: 3, + intent: { + task: "answer", + dependsOnPreviousTurn: false, + retrievalNeeded: "yes", + targetModalities: ["text"], + constraints: {}, + groundingPolicy: "must_use_sources", + }, + messages: [], + }) + + expect(result.activeTools).toEqual( + expect.arrayContaining(["memory_search", "knowhere_search"]), + ) + expect(result.activeTools).not.toContain("finalize") + expect(result.activeTools).not.toContain("knowhere_list_documents") + expect(result.activeTools).not.toContain("knowhere_get_document_outline") + expect(result.activeTools).not.toContain("knowhere_read_chunks") + expect(result.activeTools).not.toContain("knowhere_grep_chunks") + }) + + it("reopens finalize after must_use_sources has called knowhere_search, including empty results", () => { + const result = prepareHarnessStep({ + stepNumber: 4, + hasKnowhereSearch: true, + intent: { + task: "answer", + dependsOnPreviousTurn: false, + retrievalNeeded: "yes", + targetModalities: ["text"], + constraints: {}, + groundingPolicy: "must_use_sources", + }, + messages: [], + }) + + expect(result.activeTools).toContain("finalize") + expect(result.activeTools).toContain("knowhere_search") }) it("forces image inspection before forced finalization when image assets are available", () => { @@ -1033,15 +1340,19 @@ function executeTool(tool: unknown, input: unknown): Promise { return (tool as { execute: (input: unknown) => Promise }).execute(input) } +function makeMemoryTools( + search: MemoryToolRuntime["search"] = vi + .fn() + .mockResolvedValue({ query: "", items: [] }), +): MemoryToolRuntime { + return { search } +} + function makeKnowhereTools( search: KnowhereToolRuntime["search"] = vi.fn(), ): KnowhereToolRuntime { return { search, - listDocuments: vi.fn().mockResolvedValue({ documents: [] }), - getDocumentOutline: vi.fn().mockRejectedValue(new Error("Not configured.")), - readChunks: vi.fn().mockRejectedValue(new Error("Not configured.")), - grepChunks: vi.fn().mockRejectedValue(new Error("Not configured.")), } } diff --git a/src/agent-harness/runtime.ts b/src/agent-harness/runtime.ts index 073051dd..dbe91a58 100644 --- a/src/agent-harness/runtime.ts +++ b/src/agent-harness/runtime.ts @@ -10,6 +10,7 @@ import { z } from "zod" import { createEvidenceLedger } from "./ledger" import { getCanonicalImageAssetKey } from "./image-asset-identity" import { knowhereToolText } from "./knowhere-text" +import { memoryToolText } from "./memory-text" import { mergeImageInspectionHighlights } from "./image-highlights" import type { AgentTurn, @@ -17,6 +18,7 @@ import type { ContextPolicy, EvidenceAsset, EvidenceChunk, + EvidenceLedgerSnapshot, HarnessRunResult, HarnessToolCallTrace, HarnessTrace, @@ -27,8 +29,13 @@ import type { IntentFrame, KnowhereSearchTargetContent, KnowhereToolRuntime, + MemorySearchKind, + MemoryToolRuntime, + OutputArtifactView, + OutputCitation, OutputManifest, } from "./types" +import { memorySearchKinds } from "./types" const defaultMaxSteps = 14 const imageInspectionReminderStepNumber = 12 @@ -42,6 +49,7 @@ export type RunAgentHarnessInput = { readonly model: AgentHarnessModel readonly turn: AgentTurnInput readonly knowhereTools: KnowhereToolRuntime + readonly memoryTools: MemoryToolRuntime readonly inspectImages?: InspectImages readonly maxSteps?: number } @@ -62,10 +70,12 @@ type HarnessTools = ReturnType type HarnessStepPreparation = { messages: ModelMessage[] activeTools?: Array> - toolChoice?: { - type: "tool" - toolName: Extract - } + toolChoice?: + | "required" + | { + type: "tool" + toolName: Extract + } } const targetModalitySchema = z.enum(["text", "image", "table"]) @@ -77,17 +87,15 @@ const knowhereSearchTargetContentSchema = z.enum([ "text_image", "text_table", ]) -const knowledgeChunkTypeSchema = z.enum(["text", "image", "table", "page"]) - -const knowhereDocumentReferenceSchema = z.object({ - localDocumentId: z.string().min(1).optional(), - documentId: z.string().min(1).optional(), - jobId: z.string().min(1).optional(), - revisionKey: z.string().min(1).optional(), -}) const knowhereSearchSchema = z.object({ query: z.string().min(1), + includeDocumentIds: z.array(z.string().trim().min(1)).optional().describe( + "Only search these verified document IDs. Omit for unrestricted search; [] searches no documents. Use IDs from source context or previous search results, never filenames or guessed IDs.", + ), + excludeDocumentIds: z.array(z.string().trim().min(1)).optional().describe( + "Exclude these verified document IDs. Exclusions take precedence over includeDocumentIds. If the ID is unknown, describe the document constraint in query instead.", + ), targetContent: knowhereSearchTargetContentSchema.default("all"), purpose: z.string().optional(), topK: z.number().int().min(1).max(12).optional(), @@ -96,27 +104,6 @@ const knowhereSearchSchema = z.object({ threshold: z.number().min(0).max(1).optional(), }) -const knowhereReadChunksSchema = knowhereDocumentReferenceSchema.extend({ - page: z.number().int().min(1).optional(), - pageSize: z.number().int().min(1).max(50).optional(), - sectionPath: z.string().min(1).optional(), - startChunk: z.number().int().min(0).optional(), - endChunk: z.number().int().min(0).optional(), - chunkId: z.string().min(1).optional(), - chunkType: knowledgeChunkTypeSchema.optional(), -}) - -const knowhereGrepChunksSchema = knowhereDocumentReferenceSchema.extend({ - pattern: z.string().min(1), - continuationCursor: z.string().min(1).optional(), - isRegex: z.boolean().optional(), - isCaseSensitive: z.boolean().optional(), - maxResults: z.number().int().min(1).max(50).optional(), - chunkType: knowledgeChunkTypeSchema.optional(), - sectionPathPrefix: z.string().min(1).optional(), - contextChars: z.number().int().min(0).max(2_000).optional(), -}) - const intentFrameSchema = z.object({ task: z.enum([ "answer", @@ -159,16 +146,19 @@ const contextPolicySchema = z.object({ activePriorTurnIds: z.array(z.string()).default([]), }) -const outputCitationSchema = z.object({ +const citationPickSchema = z.object({ + pick: z.number().int().positive(), +}) + +const memoryCitationSchema = z.object({ ref: z.string().min(1), - label: z.string().min(1).optional(), - source: z - .object({ - documentId: z.string().nullable().optional(), - sourceFileName: z.string().nullable().optional(), - sectionPath: z.string().nullable().optional(), - }) - .optional(), + itemId: z.string().min(1), + kind: z.enum(memorySearchKinds), +}) + +const memorySearchSchema = z.object({ + query: z.string().min(1), + kinds: z.array(z.enum(memorySearchKinds)).optional(), }) const selectedOutputArtifactSchema = z.object({ @@ -194,9 +184,10 @@ const outputArtifactSchema = z.union([ derivedTableArtifactSchema, ]) -const outputManifestSchema = z.object({ +const finalizeManifestSchema = z.object({ text: z.string(), - citations: z.array(outputCitationSchema).default([]), + citations: z.array(citationPickSchema).default([]), + memoryCitations: z.array(memoryCitationSchema).default([]), artifacts: z.array(outputArtifactSchema).default([]), unresolved: z.array(z.string()).default([]), }) @@ -214,6 +205,7 @@ export async function runAgentHarness( state, ledger, knowhereTools: input.knowhereTools, + memoryTools: input.memoryTools, inspectImages: input.inspectImages, recentTurns: input.turn.recentTurns, }) @@ -225,9 +217,13 @@ export async function runAgentHarness( prepareHarnessStep({ messages: stepMessages, stepNumber, + intent: state.intent, hasUninspectedImageAssets: input.inspectImages !== undefined && hasUninspectedImageAssets({ state, ledger }), + hasKnowhereSearch: (state.toolCalls ?? []).some( + (call) => call.tool === "knowhere_search", + ), }), stopWhen: [ () => state.finalized === true, @@ -257,10 +253,36 @@ export async function runAgentHarness( } } +const alwaysAvailableTools = [ + "declareIntent", + "setContextPolicy", + "inspectImage", + "readPriorTurn", +] as const + +const fluidRetrievalTools = ["memory_search"] as const + +const crystalRetrievalTools = ["knowhere_search"] as const + +/** Reserved third retrieval slot (cognition). Not registered this round. */ +const cognitionRetrievalTools = [] as const + +// TODO(memory-architecture): today the agent itself decides, per turn via +// declareIntent, whether to call memory_search / knowhere_search as MCP +// tools. An alternative considered and deferred: always query Memento +// (including future "cognition") on every turn and let Memento decide what, +// if anything, to inject into context, instead of the agent choosing to call +// a tool. Not adopted now — it would replace this tool-invocation control +// flow with a middleware/auto-inject model and needs its own design + test +// rewrite. Revisit if agent misjudgment on retrieval-needed becomes a real +// problem. + export function prepareHarnessStep(input: { readonly stepNumber: number readonly messages: readonly ModelMessage[] readonly hasUninspectedImageAssets?: boolean + readonly hasKnowhereSearch?: boolean + readonly intent?: IntentFrame }): HarnessStepPreparation { const messages = sanitizeHarnessModelMessagesForStep(input.messages) @@ -285,24 +307,81 @@ export function prepareHarnessStep(input: { } } - if (input.stepNumber < forcedFinalizationStepNumber) { - return { messages } + if (input.stepNumber >= forcedFinalizationStepNumber) { + return { + messages: [ + ...messages, + { + role: "user", + content: buildForcedFinalizationFeedback(), + }, + ], + activeTools: ["finalize"], + toolChoice: { + type: "tool", + toolName: "finalize", + }, + } } return { - messages: [ - ...messages, - { - role: "user", - content: buildForcedFinalizationFeedback(), - }, - ], - activeTools: ["finalize"], - toolChoice: { - type: "tool", - toolName: "finalize", - }, + messages, + activeTools: selectHarnessActiveTools({ + intent: input.intent, + hasKnowhereSearch: input.hasKnowhereSearch === true, + }), + // finalize is the only output contract (see its tool description). Force + // a tool call every step so the model cannot end the turn with a bare + // text response that skips finalize's citation/artifact validation. + toolChoice: "required", + } +} + +function selectHarnessActiveTools(input: { + readonly intent?: IntentFrame + readonly hasKnowhereSearch: boolean +}): Array> { + const tools: Array> = [ + ...alwaysAvailableTools, + ] + if (allowsFinalize(input)) { + tools.push("finalize") + } + if (!allowsRetrieval(input.intent)) { + return tools + } + + // memory_search and knowhere_search are peers: both open together once + // retrieval is allowed. The agent decides which to call and in what + // order — neither tool gates the other. + tools.push(...fluidRetrievalTools) + if (input.intent?.groundingPolicy === "must_use_sources") { + tools.push(...crystalRetrievalTools) + } + tools.push(...cognitionRetrievalTools) + return tools +} + +function allowsRetrieval(intent?: IntentFrame): boolean { + if (!intent) return false + return ( + intent.groundingPolicy !== "no_retrieval" && intent.retrievalNeeded !== "no" + ) +} + +function allowsFinalize(input: { + readonly intent?: IntentFrame + readonly hasKnowhereSearch: boolean +}): boolean { + if (!input.intent) return false + if ( + input.intent.groundingPolicy === "must_use_sources" && + allowsRetrieval(input.intent) && + !input.hasKnowhereSearch + ) { + return false } + return true } export function sanitizeHarnessModelMessagesForStep( @@ -364,7 +443,7 @@ type ModelMessageForRole = Extract< function buildForcedFinalizationFeedback(): string { return [ "The retrieval step budget has been reached.", - "Do not search again or call any Knowhere evidence-reading tools.", + "Do not search again.", "Use only the evidence and tool results already available in this turn.", "Call finalize now with the best supported answer.", "If the existing evidence is insufficient, explain the gap in unresolved", @@ -410,6 +489,7 @@ export function createHarnessTools(input: { readonly state: HarnessToolState readonly ledger: ReturnType readonly knowhereTools: KnowhereToolRuntime + readonly memoryTools: MemoryToolRuntime readonly inspectImages?: InspectImages readonly recentTurns: readonly AgentTurn[] }) { @@ -446,93 +526,34 @@ export function createHarnessTools(input: { }), }), - knowhere_search: tool({ - description: - "Search Knowhere for relevant Notebook evidence. Returns tagged text with evidence refs such as r1:result:1 and asset refs such as asset:r1:result:1.", - inputSchema: knowhereSearchSchema, - execute: async (request) => - traceToolCall(input.state, { - toolName: "knowhere_search", - inputSummary: summarizeKnowhereSearchRequest(request), - execute: async () => - executeKnowhereSearch({ - ledger: input.ledger, - knowhereTools: input.knowhereTools, - request, - }), - summarizeOutput: summarizeKnowhereTextOutput, - }), - }), - - knowhere_list_documents: tool({ - description: - "List ready visible Notebook/Knowhere documents available for this chat turn. Use this to discover documentId and revisionKey before outline/read/grep.", - inputSchema: z.object({}), - execute: async () => - traceToolCall(input.state, { - toolName: "knowhere_list_documents", - inputSummary: {}, - execute: async () => - executeKnowhereTextTool({ - operation: "list_documents", - execute: async () => - knowhereToolText.formatListDocuments( - await input.knowhereTools.listDocuments(), - ), - }), - summarizeOutput: summarizeKnowhereTextOutput, - }), - }), - - knowhere_get_document_outline: tool({ + memory_search: tool({ description: - "Read a document outline from Knowhere parsed storage. Use documentId/revisionKey from knowhere_list_documents or search refs.", - inputSchema: knowhereDocumentReferenceSchema, + "Search distilled fluid memory for this workspace. Returns tagged text with memory refs such as mem:1. Use this before Knowhere document search.", + inputSchema: memorySearchSchema, execute: async (request) => traceToolCall(input.state, { - toolName: "knowhere_get_document_outline", - inputSummary: summarizeDocumentReference(request), - execute: async () => - executeKnowhereTextTool({ - operation: "get_document_outline", - validate: () => validateDocumentReference(request), - execute: async () => - knowhereToolText.formatOutline( - await input.knowhereTools.getDocumentOutline(request), - ), - }), - summarizeOutput: summarizeKnowhereTextOutput, - }), - }), - - knowhere_read_chunks: tool({ - description: - "Read complete chunk bodies from Knowhere parsed storage. This tool never slices individual chunk content; control read size with page/pageSize, sectionPath, startChunk/endChunk, chunkId, and chunkType.", - inputSchema: knowhereReadChunksSchema, - execute: async (request) => - traceToolCall(input.state, { - toolName: "knowhere_read_chunks", - inputSummary: summarizeReadChunksRequest(request), - execute: async () => - executeKnowhereReadChunks({ - ledger: input.ledger, - knowhereTools: input.knowhereTools, + toolName: "memory_search", + inputSummary: summarizeMemorySearchRequest(request), + execute: async () => { + return await executeMemorySearch({ + memoryTools: input.memoryTools, request, - }), - summarizeOutput: summarizeKnowhereTextOutput, + }) + }, + summarizeOutput: summarizeMemoryTextOutput, }), }), - knowhere_grep_chunks: tool({ + knowhere_search: tool({ description: - "Search chunk text with a literal or regex pattern. Returns bounded match snippets as grep refs such as grep1:match:1 and may include truncated=true with a continuationCursor.", - inputSchema: knowhereGrepChunksSchema, + "Search Knowhere for relevant Notebook evidence. Returns tagged text with pick numbers for finalize citations, evidence refs such as r1:result:1, and asset refs such as asset:r1:result:1.", + inputSchema: knowhereSearchSchema, execute: async (request) => traceToolCall(input.state, { - toolName: "knowhere_grep_chunks", - inputSummary: summarizeGrepChunksRequest(request), + toolName: "knowhere_search", + inputSummary: summarizeKnowhereSearchRequest(request), execute: async () => - executeKnowhereGrepChunks({ + executeKnowhereSearch({ ledger: input.ledger, knowhereTools: input.knowhereTools, request, @@ -627,16 +648,42 @@ export function createHarnessTools(input: { description: "Finalize the user-facing output manifest. This is the only final answer " + "contract. Artifacts listed here with display=true are the exact set of " + - "images/tables shown to the user; cite evidence refs when available. " + + "images/tables shown to the user. citations is the list of evidence picks " + + "you used; each pick is the pick number on a Knowhere search chunk. " + + "Notebook writes citation refs from the evidence ledger. " + + "Use memoryCitations for fluid memory refs. " + "Cited page/image assets must be inspected with inspectImage first.", - inputSchema: outputManifestSchema, + inputSchema: finalizeManifestSchema, execute: async (manifest) => traceToolCall(input.state, { toolName: "finalize", inputSummary: summarizeManifest(manifest), execute: async () => { + const resolvedCitations = resolveCitationPicks({ + citations: manifest.citations, + ledger: input.ledger, + }) + if (!resolvedCitations.ok) { + return { + ok: false as const, + message: buildFinalizeRequiresPicksMessage({ + unknownPicks: resolvedCitations.unknownPicks, + ledger: input.ledger.snapshot(), + }), + unknownPicks: resolvedCitations.unknownPicks, + } + } + + const outputManifest: OutputManifest = { + text: manifest.text, + citations: resolvedCitations.citations, + memoryCitations: manifest.memoryCitations, + artifacts: manifest.artifacts, + unresolved: manifest.unresolved, + } + const inspectRefs = getUninspectedCitedImageRefs({ - manifest, + manifest: outputManifest, ledger: input.ledger, inspectedImageRefs: input.state.inspectedImageRefs ?? [], inspectImagesAvailable: input.inspectImages !== undefined, @@ -649,9 +696,9 @@ export function createHarnessTools(input: { } } - input.state.finalizedManifest = manifest + input.state.finalizedManifest = outputManifest input.state.finalized = true - return { ok: true as const, ...manifest } + return { ok: true as const, ...outputManifest } }, summarizeOutput: summarizeFinalizeOutput, }), @@ -820,6 +867,50 @@ async function inspectRetrievedImages(input: { } } +function resolveCitationPicks(input: { + readonly citations: readonly { pick: number }[] + readonly ledger: ReturnType +}): + | { ok: true; citations: OutputCitation[] } + | { ok: false; unknownPicks: number[] } { + const chunks = input.ledger.snapshot().chunks + const unknownPicks: number[] = [] + const citations: OutputCitation[] = [] + + for (const citation of input.citations) { + const chunk = Number.isInteger(citation.pick) + ? chunks[citation.pick - 1] + : undefined + if (!chunk) { + if (!unknownPicks.includes(citation.pick)) { + unknownPicks.push(citation.pick) + } + continue + } + citations.push({ ref: chunk.ref }) + } + + if (unknownPicks.length > 0) { + return { ok: false, unknownPicks } + } + return { ok: true, citations } +} + +function buildFinalizeRequiresPicksMessage(input: { + readonly unknownPicks: readonly number[] + readonly ledger: EvidenceLedgerSnapshot +}): string { + const availableCount = input.ledger.chunks.length + return [ + "Citations must use pick numbers from Knowhere search chunks.", + `Unknown citation picks: ${input.unknownPicks.join(" ")}.`, + availableCount > 0 + ? `Available picks: 1-${availableCount}.` + : "No evidence picks are available; list the gap in unresolved and omit citations.", + "Call finalize again using only available picks.", + ].join(" ") +} + function getUninspectedCitedImageRefs(input: { readonly manifest: OutputManifest readonly ledger: ReturnType @@ -905,24 +996,27 @@ function buildFinalizeRequiresInspectionMessage( ].join(" ") } -type KnowhereToolOperation = - | "search" - | "list_documents" - | "get_document_outline" - | "read_chunks" - | "grep_chunks" +type KnowhereToolOperation = "search" +type MemorySearchToolRequest = z.infer type KnowhereSearchToolRequest = z.infer -type KnowhereDocumentReferenceRequest = z.infer< - typeof knowhereDocumentReferenceSchema -> -type KnowhereReadChunksToolRequest = z.infer -type KnowhereGrepChunksToolRequest = z.infer -type DocumentReferenceSummary = { - readonly documentId?: string - readonly localDocumentId?: string - readonly hasJobId: boolean - readonly hasRevisionKey: boolean + +async function executeMemorySearch(input: { + readonly memoryTools: MemoryToolRuntime + readonly request: MemorySearchToolRequest +}): Promise { + try { + const response = await input.memoryTools.search({ + query: input.request.query, + kinds: input.request.kinds, + }) + return memoryToolText.formatSearch(response) + } catch (error) { + return memoryToolText.formatError({ + operation: "search", + message: formatUnknownError(error), + }) + } } async function executeKnowhereSearch(input: { @@ -936,6 +1030,12 @@ async function executeKnowhereSearch(input: { const beforeSnapshot = input.ledger.snapshot() const response = await input.knowhereTools.search({ query: input.request.query, + ...(input.request.includeDocumentIds !== undefined + ? { includeDocumentIds: input.request.includeDocumentIds } + : {}), + ...(input.request.excludeDocumentIds !== undefined + ? { excludeDocumentIds: input.request.excludeDocumentIds } + : {}), targetContent: input.request.targetContent, purpose: input.request.purpose, topK: input.request.topK, @@ -947,48 +1047,7 @@ async function executeKnowhereSearch(input: { return knowhereToolText.formatSearch({ response, retrievalCount: snapshot.retrievalCount, - chunks: snapshot.chunks.slice(beforeSnapshot.chunks.length), - assets: snapshot.assets.slice(beforeSnapshot.assets.length), - }) - }, - }) -} - -async function executeKnowhereReadChunks(input: { - readonly ledger: ReturnType - readonly knowhereTools: KnowhereToolRuntime - readonly request: KnowhereReadChunksToolRequest -}): Promise { - return executeKnowhereTextTool({ - operation: "read_chunks", - validate: () => validateDocumentReference(input.request), - execute: async () => { - const beforeSnapshot = input.ledger.snapshot() - const response = await input.knowhereTools.readChunks(input.request) - const snapshot = input.ledger.addReadChunksResponse(response) - return knowhereToolText.formatReadChunks({ - response, - chunks: snapshot.chunks.slice(beforeSnapshot.chunks.length), - assets: snapshot.assets.slice(beforeSnapshot.assets.length), - }) - }, - }) -} - -async function executeKnowhereGrepChunks(input: { - readonly ledger: ReturnType - readonly knowhereTools: KnowhereToolRuntime - readonly request: KnowhereGrepChunksToolRequest -}): Promise { - return executeKnowhereTextTool({ - operation: "grep_chunks", - validate: () => validateDocumentReference(input.request), - execute: async () => { - const beforeSnapshot = input.ledger.snapshot() - const response = await input.knowhereTools.grepChunks(input.request) - const snapshot = input.ledger.addGrepChunksResponse(response) - return knowhereToolText.formatGrepChunks({ - response, + chunkPickStart: beforeSnapshot.chunks.length, chunks: snapshot.chunks.slice(beforeSnapshot.chunks.length), assets: snapshot.assets.slice(beforeSnapshot.assets.length), }) @@ -998,17 +1057,8 @@ async function executeKnowhereGrepChunks(input: { async function executeKnowhereTextTool(input: { readonly operation: KnowhereToolOperation - readonly validate?: () => string | null readonly execute: () => Promise }): Promise { - const validationError = input.validate?.() - if (validationError) { - return knowhereToolText.formatError({ - operation: input.operation, - message: validationError, - }) - } - try { return await input.execute() } catch (error) { @@ -1019,20 +1069,6 @@ async function executeKnowhereTextTool(input: { } } -function validateDocumentReference( - request: KnowhereDocumentReferenceRequest, -): string | null { - if ( - request.documentId || - request.localDocumentId || - request.jobId - ) { - return null - } - - return "A documentId, localDocumentId, or jobId is required." -} - function getUniqueTrimmedRefs(refs: readonly string[]): string[] { const normalizedRefs: string[] = [] for (const ref of refs) { @@ -1114,6 +1150,25 @@ function summarizeContextPolicy(policy: ContextPolicy): unknown { } } +function summarizeMemorySearchRequest(request: { + readonly query: string + readonly kinds?: readonly MemorySearchKind[] +}): unknown { + return { + query: request.query, + kinds: request.kinds, + } +} + +function summarizeMemoryTextOutput(output: unknown): unknown { + if (typeof output !== "string") return output + return { + ok: !output.includes('status="error"'), + textLength: output.length, + itemCount: countOccurrences(output, " artifact.display) .length, @@ -1255,6 +1275,9 @@ function summarizeFinalizeOutput(output: unknown): unknown { ok: output.ok, textLength: typeof output.text === "string" ? output.text.length : 0, citationCount: Array.isArray(output.citations) ? output.citations.length : 0, + memoryCitationCount: Array.isArray(output.memoryCitations) + ? output.memoryCitations.length + : 0, artifactCount: Array.isArray(output.artifacts) ? output.artifacts.length : 0, unresolvedCount: Array.isArray(output.unresolved) ? output.unresolved.length @@ -1282,11 +1305,17 @@ export function buildHarnessSystemPrompt(turn: AgentTurnInput): string { "1. Call declareIntent when it helps you plan the response. Capture constraints like a requested image/table count in constraints.desiredCount.", "2. Call setContextPolicy when prior turns may influence this turn.", "3. When the policy needs prior-turn detail (references or corrections), call readPriorTurn for the relevant ids.", - "4. Call knowhere_search when relevance search is needed. Use knowhere_list_documents, knowhere_get_document_outline, knowhere_read_chunks, and knowhere_grep_chunks for focused document reads.", + "4. Call memory_search first when known fluid memory may answer the request. Call knowhere_search when memory is insufficient and groundingPolicy requires citing source documents. If the returned evidence is not enough to answer, or the query needs to focus differently, call knowhere_search again with a refined query (different keywords / topK / targetContent) instead of trying to browse documents directly. Knowhere's own retrieval agent already navigates the corpus internally.", "5. After Knowhere returns image/page asset refs, call inspectImage on the page/image assets you will cite before finalize. This supplies OCR/visual context and provenance boxes.", "6. Inspect each unique cited page once; retrieval already bounds the available evidence set.", - "7. knowhere_read_chunks returns complete chunk bodies; control size with page/pageSize, sectionPath, startChunk/endChunk, chunkId, and chunkType.", - "8. Call finalize with text, citations, artifacts, and unresolved issues when you are ready to answer.", + "7. Call finalize with text, citations, artifacts, and unresolved issues when you are ready to answer.", + "", + "Retrieval rules:", + "- First use memory_search to see whether known fluid memory can answer directly.", + "- Call knowhere_search only when memory is insufficient and groundingPolicy requires citing source documents.", + "- Refine knowhere_search at most twice. If two refined searches still do not add new relevant evidence, call finalize and list the gap in unresolved.", + "- Do not treat every question as a document-retrieval task.", + "- For document-scoped searches, use includeDocumentIds/excludeDocumentIds only with verified IDs from source context or prior search results. If IDs are unknown, preserve the document requirement in query so Knowhere can locate it. Never invent IDs or substitute filenames. Exclusions win; an empty includeDocumentIds means no documents.", "", "Context rules:", "- If the current user request is unrelated to prior turns, set carryHistory to none and do not reuse prior topics.", @@ -1297,13 +1326,12 @@ export function buildHarnessSystemPrompt(turn: AgentTurnInput): string { "- Final output is the OutputManifest passed to finalize, not freeform tool JSON or trailing text.", "- artifacts with display=true are the exact images/tables shown. Never display every candidate; honor constraints.desiredCount / maxCount.", "- Use type=derived_table only for tables you create from evidence; every derived_table.sourceRefs entry must reference evidence in the ledger.", - "- Prefer citation and selected image/table artifact refs returned by Knowhere tools in the evidence ledger.", + "- citations is a list of { pick }. pick is the 1-based number on the search chunk you are using. Notebook writes the citation list from those picks. Do not pass documentId or evidence refs as citations.", "- Place [[cite:n]] immediately after the supported claim. n is the 1-based index into the citations array passed to finalize.", "- Write one marker per index: [[cite:1]] [[cite:3]] [[cite:5]]. Never group indices as [[cite:1, 3, 5]].", "- Do not write title/pN, [1], Markdown footnotes, or [Source N: ...] in the answer text. Notebook renders chips from [[cite:n]] and citation metadata.", "- Repeat [[cite:n]] when another claim uses the same page. Do not collapse same-page citations to one row.", - "- Citation label and source metadata are optional. Notebook resolves citation metadata from evidence refs when possible.", - "- If evidence is relevant but you cannot identify a supporting evidence ref, answer with unresolved issues instead of fabricating a ref.", + "- If you have no supporting evidence pick, omit citations and list the gap in unresolved.", "- inspectImage observations are inspection notes, not new source refs. Final citations and displayed image artifacts must use the original retrieved image asset refs.", "- Do not finalize cited page/image assets from chunk text alone when inspectImage is available. Inspect those asset refs first, then write the answer using the inspection notes.", "- If text evidence identifies a relevant page/image but does not include the exact fact, inspect the returned image asset for OCR/detail before saying the answer is unavailable.", @@ -1347,6 +1375,7 @@ function buildFallbackManifest(text: string): OutputManifest { return { text, citations: [], + memoryCitations: [], artifacts: [], unresolved: text ? [] : ["The agent did not finalize an output manifest."], } diff --git a/src/agent-harness/types.ts b/src/agent-harness/types.ts index edd9c2f6..b5a94168 100644 --- a/src/agent-harness/types.ts +++ b/src/agent-harness/types.ts @@ -1,10 +1,4 @@ import type { - KnowledgeDocumentReference, - KnowledgeGrepParams, - KnowledgeGrepResponse, - KnowledgeOutline, - KnowledgeReadParams, - KnowledgeReadResponse, RetrievalQueryParams, RetrievalQueryResponse, } from "@ontos-ai/knowhere-sdk" @@ -95,45 +89,59 @@ export type KnowhereSearchRequest = Pick< RetrievalQueryParams, "query" | "topK" | "signalPaths" | "filterMode" | "threshold" > & { + /** Omitted means all documents; [] means none. Exclusions take precedence. */ + readonly includeDocumentIds?: string[] + readonly excludeDocumentIds?: string[] readonly targetContent?: KnowhereSearchTargetContent readonly purpose?: string } -export type KnowhereDocumentSummary = { - readonly documentId?: string - readonly localDocumentId?: string - readonly revisionKey?: string - readonly namespace?: string - readonly sourceFileName: string - readonly title?: string - readonly status?: string - readonly chunkCount?: number - readonly typeCounts?: Readonly> -} - -export type KnowhereListDocumentsResponse = { - readonly documents: readonly KnowhereDocumentSummary[] -} - export type KnowhereToolRuntime = { readonly search: ( input: KnowhereSearchRequest, ) => Promise - readonly listDocuments: () => Promise - readonly getDocumentOutline: ( - input: KnowledgeDocumentReference, - ) => Promise - readonly readChunks: ( - input: KnowledgeReadParams, - ) => Promise - readonly grepChunks: ( - input: KnowledgeGrepParams, - ) => Promise +} + +export const memorySearchKinds = [ + "indicator_pref", + "stance", + "decision_rule", + "entity_of_interest", +] as const + +export type MemorySearchKind = (typeof memorySearchKinds)[number] + +export type MemorySearchRequest = { + readonly query: string + readonly kinds?: readonly MemorySearchKind[] +} + +export type MemorySearchItem = { + readonly ref: string + readonly itemId: string + readonly kind: MemorySearchKind + readonly abstractL0: string + readonly overviewL1: string +} + +export type MemorySearchResponse = { + readonly query: string + readonly items: readonly MemorySearchItem[] +} + +export type MemoryToolRuntime = { + readonly search: (input: MemorySearchRequest) => Promise +} + +export type MemoryCitation = { + readonly ref: string + readonly itemId: string + readonly kind: MemorySearchKind } export type EvidenceChunk = { readonly ref: string - readonly kind: "result" | "referenced_chunk" | "read_chunk" | "grep_match" + readonly kind: "result" | "referenced_chunk" readonly chunkId?: string readonly content: string readonly contentPreview: string @@ -254,6 +262,7 @@ export type OutputArtifactView = OutputArtifact | DerivedTableArtifact export type OutputManifest = { readonly text: string readonly citations: readonly OutputCitation[] + readonly memoryCitations: readonly MemoryCitation[] readonly artifacts: readonly OutputArtifactView[] readonly unresolved: readonly string[] } diff --git a/src/app/api/memory/extract/route.ts b/src/app/api/memory/extract/route.ts deleted file mode 100644 index ee25dbe4..00000000 --- a/src/app/api/memory/extract/route.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { serve } from "@upstash/workflow/nextjs" - -import { - normalizeMemoryExtractPayload, - runMemoryExtractWorkflow, - type MemoryExtractPayload, -} from "@/domains/memory/extract-workflow" -import { logger } from "@/lib/logger" - -export const { POST } = serve( - async (context) => { - const payload = normalizeMemoryExtractPayload(context.requestPayload) - if (!payload) { - logger.warn("memory: extract workflow received invalid payload") - return - } - await runMemoryExtractWorkflow({ context, payload }) - }, - { - failureFunction: async ({ context, failResponse }) => { - logger.error("memory: extract workflow failed", { - payload: context.requestPayload, - failResponse, - }) - }, - }, -) diff --git a/src/components/chat-message-list.tsx b/src/components/chat-message-list.tsx index 796f2eab..2b7a5d13 100644 --- a/src/components/chat-message-list.tsx +++ b/src/components/chat-message-list.tsx @@ -931,10 +931,14 @@ function isImageCitation( citation: ChatCitationView, assetUrl: string, ): boolean { - return ( - citation.chunkType.toLowerCase() === "image" || - hasImageFileExtension(assetUrl) - ); + // chunkType is sourced from Knowhere retrieval results, which can omit it + // at runtime even though the type says it's a required string. Normalize + // defensively instead of calling .toLowerCase() on a possibly-missing value. + const chunkType = + typeof citation.chunkType === "string" + ? citation.chunkType.toLowerCase() + : ""; + return chunkType === "image" || hasImageFileExtension(assetUrl); } function hasImageFileExtension(assetUrl: string): boolean { diff --git a/src/domains/chat/citations.test.ts b/src/domains/chat/citations.test.ts index b381a83a..1335de3c 100644 --- a/src/domains/chat/citations.test.ts +++ b/src/domains/chat/citations.test.ts @@ -64,6 +64,21 @@ describe("toChatCitationViews", () => { expect(citations[0]?.pageCitationPageNumber).toBe(26) }) + it("copies the chunk id onto the citation when the retrieval result has one", () => { + const citations = toChatCitationViews( + [makeRetrievalResult({ chunkId: "chunk_123" })], + "Grounded answer.", + ) + + expect(citations[0]?.chunkId).toBe("chunk_123") + }) + + it("omits chunkId when the retrieval result has none", () => { + const citations = toChatCitationViews([makeRetrievalResult()], "Grounded answer.") + + expect(citations[0]).not.toHaveProperty("chunkId") + }) + it("copies inspect-image provenance boxes onto the citation", () => { const citations = toChatCitationViews( [ diff --git a/src/domains/chat/citations.ts b/src/domains/chat/citations.ts index 9fb6c8dc..18304aa4 100644 --- a/src/domains/chat/citations.ts +++ b/src/domains/chat/citations.ts @@ -20,6 +20,7 @@ export function toChatCitationViews( content: result.content, chunkType: result.chunkType, score: result.score, + ...(result.chunkId ? { chunkId: result.chunkId } : {}), ...(result.assetUrl ? { assetUrl: result.assetUrl } : {}), ...(result.pageCitationAssetUrl ? { pageCitationAssetUrl: result.pageCitationAssetUrl } diff --git a/src/domains/chat/commit-turn.test.ts b/src/domains/chat/commit-turn.test.ts new file mode 100644 index 00000000..97f19dae --- /dev/null +++ b/src/domains/chat/commit-turn.test.ts @@ -0,0 +1,135 @@ +import { Either } from "effect" +import { beforeEach, describe, expect, it, vi } from "vitest" + +const mocks = vi.hoisted(() => ({ + handleChatTurn: vi.fn(), + captureMemoryTurn: vi.fn(), + recordActivations: vi.fn(), +})) + +vi.mock("./service", () => ({ + handleChatTurn: mocks.handleChatTurn, +})) + +vi.mock("@/integrations/memento/client", () => ({ + captureMemoryTurn: mocks.captureMemoryTurn, + recordActivations: mocks.recordActivations, +})) + +import { commitChatTurn } from "./commit-turn" +import type { Workspace } from "@/infrastructure/db/schema" + +describe("commitChatTurn", () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.recordActivations.mockResolvedValue(undefined) + mocks.captureMemoryTurn.mockResolvedValue(undefined) + }) + + it("records fluid memory activations from finalize after a successful turn", async () => { + mocks.handleChatTurn.mockImplementation(async (input) => { + await input.generateAnswer({ + question: "毛利率", + messages: [], + sources: [], + excludedSourceIds: [], + searchSources: vi.fn(), + }) + return Either.right({ + threadId: "thread_1", + messages: [ + { id: "msg_user", role: "user", content: "毛利率" }, + { + id: "msg_assistant", + role: "assistant", + content: "按已有记忆。", + citations: [ + { + chunkId: "chunk_1", + chunkType: "text", + score: 0.9, + source: { documentId: "doc_1" }, + }, + ], + }, + ], + }) + }) + + const generateAnswer = vi.fn(async () => ({ + manifest: { + text: "按已有记忆。", + citations: [], + memoryCitations: [ + { ref: "mem:1", itemId: "item_1", kind: "stance" as const }, + ], + artifacts: [], + unresolved: [], + }, + trace: { + ledger: { + retrievalCount: 0, + chunks: [], + assets: [], + evidenceText: [], + stopReasons: [], + failureReasons: [], + decisionTraces: [], + }, + finalized: true, + priorTurnReads: [], + toolCalls: [], + imageHighlights: [], + validationErrors: [], + revisionsUsed: 0, + }, + })) + + const result = await commitChatTurn({ + workspace: makeWorkspace(), + sources: [], + question: "毛利率", + excludedSourceIds: [], + retrieval: { query: vi.fn() }, + generateAnswer, + repository: { + ensureDefaultChatThread: vi.fn(), + findChatThreadInWorkspace: vi.fn(), + listMessagesForThread: vi.fn(), + appendMessageToThread: vi.fn(), + }, + }) + + expect(Either.isRight(result)).toBe(true) + expect(mocks.captureMemoryTurn).toHaveBeenCalledWith({ + workspaceId: "workspace_1", + sourceMessageId: "msg_assistant", + userText: "毛利率", + assistantText: "按已有记忆。", + referencedDocumentIds: ["doc_1"], + }) + expect(mocks.recordActivations).toHaveBeenCalledWith([ + { + workspaceId: "workspace_1", + unitType: "crystal_chunk", + unitRef: "doc_1:chunk_1", + }, + ]) + expect(mocks.recordActivations).toHaveBeenCalledWith([ + { + workspaceId: "workspace_1", + unitType: "fluid_memory", + unitRef: "item_1", + }, + ]) + }) +}) + +function makeWorkspace(): Workspace { + return { + id: "workspace_1", + userId: "user_1", + namespace: "notebook-namespace", + createdAt: new Date("2026-09-10T00:00:00Z"), + } +} diff --git a/src/domains/chat/commit-turn.ts b/src/domains/chat/commit-turn.ts new file mode 100644 index 00000000..1113058b --- /dev/null +++ b/src/domains/chat/commit-turn.ts @@ -0,0 +1,133 @@ +import { Either } from "effect" + +import type { MemoryCitation } from "@/agent-harness" +import { + captureMemoryTurn, + recordActivations, +} from "@/integrations/memento/client" +import { generateAgenticOutputManifest } from "./prompt" +import type { ChatCitationView } from "./types" +import { + handleChatTurn, + type ChatTurnError, + type ChatTurnValue, +} from "./service" + +type CommitChatTurnInput = Parameters[0] + +/** + * Production chat-turn commit used by the HTTP route: answer → persist → + * capture fluid memory on Memento → record cited crystal/memory activations. + */ +export async function commitChatTurn( + input: CommitChatTurnInput, +): Promise> { + let memoryCitations: readonly MemoryCitation[] = [] + const result = await handleChatTurn({ + ...input, + generateAnswer: async (generateInput) => { + const generated = await input.generateAnswer({ + ...generateInput, + }) + memoryCitations = generated.manifest.memoryCitations + return generated + }, + }) + + if (Either.isRight(result)) { + const [userMessage, assistantMessage] = result.right.messages + void captureMemoryTurn({ + workspaceId: input.workspace.id, + sourceMessageId: assistantMessage.id, + userText: userMessage.content, + assistantText: assistantMessage.content, + referencedDocumentIds: collectCitationDocumentIds( + assistantMessage.citations, + ), + }) + void recordChunkActivations({ + workspaceId: input.workspace.id, + citations: assistantMessage.citations, + }) + void recordMemoryActivations({ + workspaceId: input.workspace.id, + memoryCitations, + }) + } + + return result +} + +export async function commitAgenticChatTurn( + input: Omit, +): Promise> { + return commitChatTurn({ + ...input, + generateAnswer: (generateInput) => + generateAgenticOutputManifest({ + ...generateInput, + workspaceId: input.workspace.id, + }), + }) +} + +/** + * Fire-and-forget activation ledger write for crystal chunks actually cited + * in this turn's answer. Only citations with both a documentId and a + * chunkId count — a citation missing either can't be identified down to a + * chunk (see ChatCitationView / RetrievalResultView). + */ +export async function recordChunkActivations(input: { + readonly workspaceId: string + readonly citations: readonly ChatCitationView[] | undefined +}): Promise { + const activationInputs = (input.citations ?? []).flatMap((citation) => { + const documentId = citation.source.documentId + const chunkId = citation.chunkId + if (!documentId || !chunkId) return [] + return [ + { + workspaceId: input.workspaceId, + unitType: "crystal_chunk" as const, + unitRef: toChunkUnitRef({ documentId, chunkId }), + }, + ] + }) + await recordActivations(activationInputs) +} + +/** + * Fire-and-forget activation ledger write for fluid memory items actually + * cited in this turn's finalize output. + */ +export async function recordMemoryActivations(input: { + readonly workspaceId: string + readonly memoryCitations: readonly MemoryCitation[] +}): Promise { + const activationInputs = input.memoryCitations.map((citation) => ({ + workspaceId: input.workspaceId, + unitType: "fluid_memory" as const, + unitRef: citation.itemId, + })) + await recordActivations(activationInputs) +} + +function toChunkUnitRef(input: { + readonly documentId: string + readonly chunkId: string +}): string { + return `${input.documentId}:${input.chunkId}` +} + +function collectCitationDocumentIds( + citations: readonly ChatCitationView[] | undefined, +): string[] { + const ids = new Set() + for (const citation of citations ?? []) { + const documentId = citation.source.documentId + if (typeof documentId === "string" && documentId.length > 0) { + ids.add(documentId) + } + } + return [...ids] +} diff --git a/src/domains/chat/contracts.ts b/src/domains/chat/contracts.ts index 527b52c4..a38a6f98 100644 --- a/src/domains/chat/contracts.ts +++ b/src/domains/chat/contracts.ts @@ -9,6 +9,7 @@ import type { HarnessRunResult, InspectImages, KnowhereToolRuntime, + KnowhereSearchRequest, } from "@/agent-harness" import type { ChatArtifactView, @@ -16,7 +17,6 @@ import type { } from "@/domains/chat/types" import type { HardenMediaAssetUrls } from "./media-asset-hardening" import type { HardenChatAssetUrl } from "./media-assets" -import type { NotebookKnowhereRemoteDocumentClient } from "./knowhere-tools" export type RetrievalClient = { query(params: RetrievalQueryParams): Promise @@ -41,13 +41,7 @@ export type AgenticRetrievalPlan = { purpose: string | null } -export type AgenticRetrievalQuery = Pick< - RetrievalQueryParams, - "query" | "topK" | "signalPaths" | "filterMode" | "threshold" -> & { - readonly targetContent?: AgenticRetrievalTargetContent - readonly purpose?: string -} +export type AgenticRetrievalQuery = KnowhereSearchRequest export type AgenticRetrievalResponse = RetrievalQueryResponse & { retrievalPlan?: AgenticRetrievalPlan @@ -76,7 +70,6 @@ export type AnswerQuestionInput = { useAgentic?: boolean retrieval: RetrievalClient knowledge?: Knowledge - remoteDocumentClient?: NotebookKnowhereRemoteDocumentClient generateAnswer: GenerateAnswer hardenChatAssetUrl?: HardenChatAssetUrl hardenMediaAssetUrls?: HardenMediaAssetUrls diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index 1bc9b606..26fd0385 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -1,9 +1,9 @@ import { afterEach, describe, expect, it, vi } from "vitest" +import { createServer } from "node:http" +import { once } from "node:events" +import Knowhere from "@ontos-ai/knowhere-sdk" import type { Knowledge, - KnowledgeGrepResponse, - KnowledgeOutline, - KnowledgeReadResponse, RetrievalQueryParams, RetrievalQueryResponse, RetrievalResult, @@ -106,6 +106,94 @@ describe("answerQuestionWithRetrieval", () => { }); }); + it.each([true, false])("sends document scope through the installed SDK (agentic=%s)", async (useAgentic) => { + const received: Record[] = [] + const result = makeRetrievalResult({ + content: "Evidence from the allowed document.", + source: { documentId: "doc_included", sourceFileName: "notes.txt", sectionPath: "Overview" }, + }) + const server = createServer((request, response) => { + let body = "" + request.setEncoding("utf8") + request.on("data", (chunk: string) => { body += chunk }) + request.on("end", () => { + received.push(JSON.parse(body)) + response.writeHead(200, { "Content-Type": "application/json" }) + response.end(JSON.stringify({ + namespace: "default", query: "document question", router_used: "test_fixture", + results: [{ content: result.content, chunk_type: "text", score: 1, + source: { document_id: "doc_included", source_file_name: "notes.txt", section_path: "Overview" } }], + referenced_chunks: [], evidence_text: result.content, answer_text: "", + })) + }) + }) + try { + server.listen(0, "127.0.0.1") + await once(server, "listening") + const address = server.address() + if (!address || typeof address === "string") throw new Error("Expected TCP address") + const client = new Knowhere({ apiKey: "scope-test", baseURL: `http://127.0.0.1:${address.port}`, maxRetries: 0 }) + const scopes = [ + {}, + { includeDocumentIds: ["doc_included", "doc_other"] }, + { excludeDocumentIds: ["doc_other"] }, + { includeDocumentIds: ["doc_included", "doc_other"], excludeDocumentIds: ["doc_other"] }, + { includeDocumentIds: [] }, + ] + const answer = await Effect.runPromise(answerQuestionWithRetrieval({ + question: "document question", namespace: "default", useAgentic, + sources: [makeSource(), makeSource({ id: "source_other", knowhereDocumentId: "doc_other" })], + excludedSourceIds: ["knowhere-doc:default:doc_user_excluded"], + retrieval: client.retrieval, + generateAnswer: async ({ knowhereTools }) => { + if (!knowhereTools) throw new Error("Missing Knowhere tools") + for (const scope of scopes) await knowhereTools.search({ query: "document question", ...scope }) + return makeCitedHarnessRunResult("Evidence [[cite:1]].", result) + }, + messages: [], + })) + expect(received).toEqual(scopes.map((scope) => ({ + namespace: "default", query: "document question", top_k: 8, use_agentic: useAgentic, data_type: 1, + ...(scope.includeDocumentIds !== undefined ? { include_document_ids: scope.includeDocumentIds } : {}), + exclude_document_ids: ["doc_user_excluded", ...(scope.excludeDocumentIds ?? [])], + }))) + expect(answer.answer).toBe("Evidence [[cite:1]].") + expect(answer.citations).toHaveLength(1) + expect(answer.citations[0]?.source.documentId).toBe("doc_included") + } finally { + if (server.listening) { + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()) + server.closeAllConnections() + }) + } + } + }) + + it("rejects invented document IDs but accepts IDs discovered by a prior search", async () => { + const result = makeRetrievalResult({ source: { + documentId: "doc_discovered", sourceFileName: "named-document.pdf", sectionPath: "Overview", + } }) + const retrieval = { query: vi.fn().mockResolvedValue({ + namespace: "default", query: "named document", routerUsed: "agent_explore", + results: [result], referencedChunks: [], evidenceText: result.content, answerText: "", + }) } + await Effect.runPromise(answerQuestionWithRetrieval({ + question: "Only search the named document", namespace: "default", + sources: [makeSource()], excludedSourceIds: [], retrieval, messages: [], + generateAnswer: async ({ knowhereTools }) => { + if (!knowhereTools) throw new Error("Missing Knowhere tools") + await expect(knowhereTools.search({ query: "question", includeDocumentIds: ["named-document.pdf"] })).rejects.toThrow("unverified ID") + await expect(knowhereTools.search({ query: "question", excludeDocumentIds: ["doc_invented"] })).rejects.toThrow("unverified ID") + expect(retrieval.query).not.toHaveBeenCalled() + await knowhereTools.search({ query: "Only search named-document.pdf" }) + await knowhereTools.search({ query: "question", includeDocumentIds: ["doc_discovered"] }) + expect(retrieval.query).toHaveBeenLastCalledWith(expect.objectContaining({ includeDocumentIds: ["doc_discovered"] })) + return makeCitedHarnessRunResult("Evidence [[cite:1]].", result) + }, + })) + }) + it("does not create source chips from retrieval results when the manifest has no citations", async () => { const unrelatedResult = makeRetrievalResult({ content: "Information hiding is unrelated to the requested source.", @@ -150,7 +238,7 @@ describe("answerQuestionWithRetrieval", () => { }); }); - it("exposes search, list, outline, read, and grep through the Knowhere tool runtime", async () => { + it("exposes search through the Knowhere tool runtime", async () => { const result = makeRetrievalResult({ chunkType: "image", source: { @@ -170,36 +258,6 @@ describe("answerQuestionWithRetrieval", () => { answerText: null, }), }; - const getDocumentOutline = vi.fn().mockResolvedValue(makeKnowledgeOutline()); - const readChunks = vi.fn().mockResolvedValue( - makeKnowledgeReadResponse("Full diagram chunk body."), - ); - const grepChunks = vi.fn().mockResolvedValue(makeKnowledgeGrepResponse()); - const knowledge = { - getDocumentOutline, - readChunks, - grepChunks, - } as unknown as Knowledge; - const listDocuments = vi.fn().mockResolvedValue({ - documents: [ - { - documentId: "doc_remote", - namespace: "default", - status: "ready", - currentJobResultId: "job_remote", - sourceFileName: "remote.pdf", - documentMetadata: { - createdByClient: "cli", - }, - }, - { - documentId: "doc_untagged", - namespace: "default", - status: "ready", - sourceFileName: "dummy.pdf", - }, - ], - }); const generateAnswer = vi.fn( async ({ knowhereTools }: Parameters[0]) => { if (!knowhereTools) throw new Error("Knowhere tools were not provided."); @@ -209,28 +267,8 @@ describe("answerQuestionWithRetrieval", () => { targetContent: "image", topK: 2, }); - const documents = await knowhereTools.listDocuments(); - await knowhereTools.getDocumentOutline({ - documentId: "doc_included", - revisionKey: "job_123", - }); - await knowhereTools.readChunks({ - documentId: "doc_included", - revisionKey: "job_123", - page: 1, - pageSize: 2, - }); - await knowhereTools.grepChunks({ - documentId: "doc_included", - revisionKey: "job_123", - pattern: "diagram", - maxResults: 3, - }); expect(searchResponse.results).toEqual([result]); - expect( - documents.documents.map((document) => document.documentId), - ).toEqual(["doc_included", "doc_remote"]); return makeHarnessRunResult("Runtime answer."); }, ); @@ -246,8 +284,6 @@ describe("answerQuestionWithRetrieval", () => { sources, excludedSourceIds: ["source_excluded"], retrieval, - knowledge, - remoteDocumentClient: { documents: { list: listDocuments } }, generateAnswer, messages: [], }), @@ -261,27 +297,6 @@ describe("answerQuestionWithRetrieval", () => { dataType: 3, excludeDocumentIds: ["doc_excluded"], }); - expect(listDocuments).toHaveBeenCalledWith({ - namespace: "default", - page: 1, - pageSize: 200, - }); - expect(getDocumentOutline).toHaveBeenCalledWith({ - documentId: "doc_included", - revisionKey: "job_123", - }); - expect(readChunks).toHaveBeenCalledWith({ - documentId: "doc_included", - revisionKey: "job_123", - page: 1, - pageSize: 2, - }); - expect(grepChunks).toHaveBeenCalledWith({ - documentId: "doc_included", - revisionKey: "job_123", - pattern: "diagram", - maxResults: 3, - }); expect(answer.answer).toBe("Runtime answer."); }); @@ -1040,6 +1055,7 @@ describe("answerQuestionWithRetrieval", () => { manifest: { text: `Use this image. ${rawAssetUrl}`, citations: [], + memoryCitations: [], artifacts: [ { type: "image", @@ -1430,10 +1446,10 @@ describe("answerQuestionWithRetrieval", () => { expect(answer.citations[0]?.pageCitationAssetUrl).toBe(hardenedPageAssetUrl); }); - it("hydrates page numbers for grep citations from the matching parsed chunk", async () => { + it("hydrates page numbers for citations missing page metadata from the matching parsed chunk", async () => { const grepChunk = { - ref: "grep1:match:1", - kind: "grep_match" as const, + ref: "r1:result:1", + kind: "result" as const, chunkId: "chunk_financial_summary", content: "ept percentages and per share data)\nTotal automotive revenues\n17,693", contentPreview: "ept percentages and per share data)", @@ -1478,7 +1494,7 @@ describe("answerQuestionWithRetrieval", () => { makeHarnessRunResultWithLedger( "Automotive revenue was $17,693 million [[cite:1]].", { - citations: [{ ref: "grep1:match:1" }], + citations: [{ ref: "r1:result:1" }], chunks: [grepChunk], }, ), @@ -1773,6 +1789,7 @@ describe("answerQuestionWithRetrieval", () => { manifest: { text: "已找到相关身份证图片，见下方图片。", citations: [], + memoryCitations: [], artifacts: [ { type: "image", @@ -2052,17 +2069,8 @@ describe("answerQuestionWithRetrieval", () => { await tools.finalize?.execute({ text: "Information hiding is a module design principle.", - citations: [ - { - ref: "r1:result:1", - label: "claimed-source.pdf / Claimed", - source: { - documentId: "doc_claimed", - sourceFileName: "claimed-source.pdf", - sectionPath: "Claimed", - }, - }, - ], + citations: [{ pick: 1 }], + memoryCitations: [], artifacts: [], unresolved: [], }); @@ -2081,7 +2089,11 @@ describe("answerQuestionWithRetrieval", () => { sources: [makeSource()], excludedSourceIds: [], retrieval, - generateAnswer: generateAgenticOutputManifest, + generateAnswer: (input) => + generateAgenticOutputManifest({ + ...input, + workspaceId: "workspace_1", + }), messages: [], }), ); @@ -2128,6 +2140,7 @@ describe("answerQuestionWithRetrieval", () => { manifest: { text: "", citations: [], + memoryCitations: [], artifacts: [ { type: "image", @@ -2243,6 +2256,7 @@ describe("answerQuestionWithRetrieval", () => { manifest: { text: "I organized the comparison into a table.", citations: [], + memoryCitations: [], artifacts: [ { type: "derived_table", @@ -2639,6 +2653,7 @@ describe("answerQuestionWithRetrieval", () => { content: "", chunkType: "image", score: null, + chunkId: "chunk_1", assetUrl: "https://blob.example/images/launch.jpg", source: { documentId: "doc_spacex", @@ -2688,17 +2703,8 @@ describe("generateAgenticOutputManifest", () => { }); await tools.finalize?.execute({ text: "已找到相关身份证图片，见下方图片。", - citations: [ - { - ref: "r1:result:1", - label: "商务标文件.pdf / 身份证正面", - source: { - documentId: "doc_identity", - sourceFileName: "document-generated.pdf", - sectionPath: "身份证正面", - }, - }, - ], + citations: [{ pick: 1 }], + memoryCitations: [], artifacts: [ { type: "image", @@ -2739,6 +2745,7 @@ describe("generateAgenticOutputManifest", () => { }); const result = await generateAgenticOutputManifest({ + workspaceId: "workspace_1", question: "请只返回冯荣洲的 2 张身份证图片", messages: [ { @@ -2825,17 +2832,8 @@ describe("generateAgenticOutputManifest", () => { }); await tools.finalize?.execute({ text: "The inspected image appears to show the requested ID card.", - citations: [ - { - ref: "asset:r1:result:1", - label: "identity.pdf / images/id-front.png", - source: { - documentId: "doc_identity", - sourceFileName: "generated.pdf", - sectionPath: "images/id-front.png", - }, - }, - ], + citations: [{ pick: 1 }], + memoryCitations: [], artifacts: [ { type: "image", @@ -2885,6 +2883,7 @@ describe("generateAgenticOutputManifest", () => { }); const result = await generateAgenticOutputManifest({ + workspaceId: "workspace_1", question: "Inspect and show the ID card image.", messages: [], sources: [ @@ -2912,7 +2911,7 @@ describe("generateAgenticOutputManifest", () => { ], }); expect(result.manifest.citations.map((citation) => citation.ref)).toEqual([ - "asset:r1:result:1", + "r1:result:1", ]); expect(result.manifest.artifacts).toEqual([ { @@ -2965,17 +2964,8 @@ describe("generateAgenticOutputManifest", () => { }); await tools.finalize?.execute({ text: "承包人自行修改发包人审批的进度计划，应按每次 5000 元赔偿违约金。", - citations: [ - { - ref: "asset:r1:referenced:1", - label: "投标书 / （6）现场工期进度管理方面的违约责任", - source: { - documentId: "doc_contract", - sourceFileName: null, - sectionPath: "Root / （6）现场工期进度管理方面的违约责任", - }, - }, - ], + citations: [{ pick: 1 }], + memoryCitations: [], artifacts: [], unresolved: [], }); @@ -3029,6 +3019,7 @@ describe("generateAgenticOutputManifest", () => { }); const result = await generateAgenticOutputManifest({ + workspaceId: "workspace_1", question: "承包人自行修改发包人审批的进度时需要赔偿多少违约金？", messages: [], sources: [ @@ -3072,7 +3063,7 @@ describe("generateAgenticOutputManifest", () => { }); expect(result.manifest.text).toContain("5000 元"); expect(result.manifest.citations.map((citation) => citation.ref)).toEqual([ - "asset:r1:referenced:1", + "r1:referenced:1", ]); expect(result.trace.toolCalls.map((call) => call.tool)).toContain( "inspectImage", @@ -3115,17 +3106,8 @@ describe("generateAgenticOutputManifest", () => { }); await tools.finalize?.execute({ text: "见下方图片。", - citations: [ - { - ref: "r1:result:1", - label: "ids.pdf / 身份证 1", - source: { - documentId: "doc_identity", - sourceFileName: "ids.pdf", - sectionPath: "身份证 1", - }, - }, - ], + citations: [{ pick: 1 }], + memoryCitations: [], artifacts: [1, 2, 3].map((index) => ({ type: "image", ref: `asset:r1:result:${index}`, @@ -3137,17 +3119,8 @@ describe("generateAgenticOutputManifest", () => { } else { await tools.finalize?.execute({ text: "见下方图片。", - citations: [ - { - ref: "r1:result:1", - label: "ids.pdf / 身份证 1", - source: { - documentId: "doc_identity", - sourceFileName: "ids.pdf", - sectionPath: "身份证 1", - }, - }, - ], + citations: [{ pick: 1 }], + memoryCitations: [], artifacts: [1, 2].map((index) => ({ type: "image", ref: `asset:r1:result:${index}`, @@ -3188,6 +3161,7 @@ describe("generateAgenticOutputManifest", () => { }); const result = await generateAgenticOutputManifest({ + workspaceId: "workspace_1", question: "只要 2 张身份证图片", messages: [], sources: [ @@ -3322,6 +3296,7 @@ function makeHarnessRunResult(text: string): HarnessRunResult { manifest: { text, citations: [], + memoryCitations: [], artifacts: [], unresolved: [], }, @@ -3371,6 +3346,7 @@ function makeHarnessRunResultWithLedger( manifest: { text, citations: input.citations ?? [], + memoryCitations: [], artifacts: input.artifacts ?? [], unresolved: [], }, @@ -3431,87 +3407,6 @@ function makeEvidenceChunkFromRetrievalResult( }; } -function makeKnowledgeOutline(): KnowledgeOutline { - return { - document: makeLocalKnowledgeDocument(), - totalChunks: 1, - typeCounts: { text: 1, image: 0, table: 0, page: 0 }, - sections: [ - { - sectionPath: "Root / Diagram", - sectionTitle: "Diagram", - sectionLevel: 2, - summary: "Diagram section.", - startChunk: 1, - endChunk: 1, - chunkCount: 1, - typeCounts: { text: 1, image: 0, table: 0, page: 0 }, - children: [], - }, - ], - sectionTree: [], - }; -} - -function makeKnowledgeReadResponse(content: string): KnowledgeReadResponse { - return { - document: makeLocalKnowledgeDocument(), - chunks: [ - { - position: 1, - chunkId: "chunk_1", - chunkType: "text", - content, - readableContent: content, - sectionPath: "Root / Diagram", - sourceChunkPath: "chunks/chunk-1.md", - filePath: "notes.txt", - metadata: {}, - }, - ], - page: 1, - pageSize: 1, - totalChunks: 1, - totalPages: 1, - }; -} - -function makeKnowledgeGrepResponse(): KnowledgeGrepResponse { - return { - document: makeLocalKnowledgeDocument(), - matches: [ - { - position: 1, - chunkId: "chunk_1", - chunkType: "text", - sectionPath: "Root / Diagram", - sourceChunkPath: "chunks/chunk-1.md", - filePath: "notes.txt", - startOffset: 0, - endOffset: 7, - snippet: "diagram", - }, - ], - scannedChunks: 1, - truncated: false, - }; -} - -function makeLocalKnowledgeDocument() { - return { - localDocumentId: "doc_included", - documentId: "doc_included", - jobId: "job_123", - namespace: "notebook-workspace", - sourceFileName: "notes.txt", - chunkCount: 1, - typeCounts: { text: 1, image: 0, table: 0, page: 0 }, - resultDirectoryPath: "parsed-storage:doc_included/job_123", - createdAt: new Date("2026-01-01T00:00:00Z"), - updatedAt: new Date("2026-01-01T00:00:00Z"), - }; -} - type KnowhereQueryResponseLogMeta = { readonly query: string readonly resultCount: number diff --git a/src/domains/chat/index.ts b/src/domains/chat/index.ts index d2b7208f..9f73b20b 100644 --- a/src/domains/chat/index.ts +++ b/src/domains/chat/index.ts @@ -32,7 +32,7 @@ import type { AnswerQuestionResult, } from "./contracts" import { - excludeDocuments, + getRetrievalDocumentScope, normalizeRetrievalQuery, } from "./retrieval" import { @@ -247,12 +247,8 @@ export const answerQuestionWithRetrieval = ( excludedSourceIds: input.excludedSourceIds, searchSources, knowhereTools: notebookKnowhereTools.createRuntime({ - namespace: input.namespace, - sources: input.sources, - excludedSourceIds: input.excludedSourceIds, searchSources, - knowledge: input.knowledge, - remoteDocumentClient: input.remoteDocumentClient, + sources: input.sources, }), ...(input.inspectImages ? { inspectImages: input.inspectImages } : {}), }), @@ -291,7 +287,6 @@ export const answerQuestionWithRetrieval = ( results: useNotebookSourceTitles(rawResults, input.sources), sources: input.sources, hardenChatAssetUrl: input.hardenChatAssetUrl, - evidenceText: formatRetrievalEvidenceText(retrievalResponses), }), ) const pageCitationResults = yield* Effect.tryPromise(() => @@ -982,7 +977,7 @@ function buildRetrievalQueryParams(input: { ...(typeof input.input.threshold === "number" ? { threshold: input.input.threshold } : {}), - ...excludeDocuments(input.sources, input.excludedSourceIds), + ...getRetrievalDocumentScope(input.sources, input.excludedSourceIds, input.input), } } @@ -1057,12 +1052,16 @@ function mapManifestCitationsToResults( ) const results: RetrievalResult[] = [] + const droppedRefs: string[] = [] for (const citation of result.manifest.citations) { const chunk = chunksByRef.get(citation.ref) ?? resolveChunkForAssetRef(citation.ref, assetsByRef, chunksByRef) - if (!chunk) continue + if (!chunk) { + droppedRefs.push(citation.ref) + continue + } const retrievalResult = toRetrievalResultFromEvidenceChunk( mergeChunkPageMetadata(chunk, result.trace.ledger.chunks), @@ -1081,6 +1080,14 @@ function mapManifestCitationsToResults( if (results.length >= MAX_CITATION_RESULTS) break } + if (droppedRefs.length > 0) { + logger.warn("chat-agent: dropped unresolved citation refs", { + droppedRefs, + ledgerChunkRefs: result.trace.ledger.chunks.map((chunk) => chunk.ref), + ledgerAssetRefs: result.trace.ledger.assets.map((asset) => asset.ref), + }) + } + return results } @@ -1343,17 +1350,6 @@ function hasDisplayedManifestArtifacts(result: HarnessRunResult): boolean { return result.manifest.artifacts.some((artifact) => artifact.display) } -function formatRetrievalEvidenceText( - responses: readonly RetrievalQueryResponse[], -): string | undefined { - const evidenceText = responses - .map((response): string => response.evidenceText?.trim() ?? "") - .filter((value): boolean => value.length > 0) - .join("\n") - - return evidenceText || undefined -} - function getRetrievalResultKey(result: RetrievalResult): string { const source = result.source return [ diff --git a/src/domains/chat/knowhere-tools.ts b/src/domains/chat/knowhere-tools.ts index 5c70fd7b..49bdbb92 100644 --- a/src/domains/chat/knowhere-tools.ts +++ b/src/domains/chat/knowhere-tools.ts @@ -1,157 +1,39 @@ -import { Effect } from "effect" -import type { Knowledge } from "@ontos-ai/knowhere-sdk" - -import type { - KnowhereDocumentSummary, - KnowhereToolRuntime, -} from "@/agent-harness" -import type { Source } from "@/infrastructure/db/schema" -import { - listRemoteLibraryDocuments, - isNotebookVisibleRemoteDocument, - type RemoteLibraryDocument, -} from "@/domains/sources/remote-library" +import type { KnowhereToolRuntime } from "@/agent-harness" import type { SearchSources } from "./contracts" -import { excludeDocuments } from "./retrieval" - -type RemoteDocumentClient = Parameters< - typeof listRemoteLibraryDocuments ->[0]["client"] - -export type NotebookKnowhereRemoteDocumentClient = RemoteDocumentClient +import type { Source } from "@/infrastructure/db/schema" type NotebookKnowhereToolsInput = { - readonly namespace: string - readonly sources: readonly Source[] - readonly excludedSourceIds: readonly string[] - readonly searchSources: SearchSources - readonly knowledge?: Knowledge - readonly remoteDocumentClient?: RemoteDocumentClient -} - -type SearchOnlyRuntimeInput = { readonly searchSources: SearchSources + readonly sources: readonly Source[] } export const notebookKnowhereTools = { createRuntime(input: NotebookKnowhereToolsInput): KnowhereToolRuntime { + const knownDocumentIds = new Set( + input.sources.flatMap((source) => + source.knowhereDocumentId ? [source.knowhereDocumentId] : [], + ), + ) return { - search: (request) => input.searchSources(request), - listDocuments: async () => ({ - documents: await listVisibleDocuments(input), - }), - getDocumentOutline: async (request) => { - const knowledge = requireKnowledge(input.knowledge) - return knowledge.getDocumentOutline(request) - }, - readChunks: async (request) => { - const knowledge = requireKnowledge(input.knowledge) - return knowledge.readChunks(request) - }, - grepChunks: async (request) => { - const knowledge = requireKnowledge(input.knowledge) - return knowledge.grepChunks(request) - }, - } - }, - - createSearchOnlyRuntime(input: SearchOnlyRuntimeInput): KnowhereToolRuntime { - return { - search: (request) => input.searchSources(request), - listDocuments: async () => ({ documents: [] }), - getDocumentOutline: async () => { - throw new Error("Knowhere document outline is not configured.") - }, - readChunks: async () => { - throw new Error("Knowhere chunk reads are not configured.") - }, - grepChunks: async () => { - throw new Error("Knowhere grep is not configured.") + search: async (request) => { + const requestedIds = [ + ...(request.includeDocumentIds ?? []), + ...(request.excludeDocumentIds ?? []), + ] + if (requestedIds.some((id) => !knownDocumentIds.has(id))) { + throw new Error( + "Document scope contains an unverified ID. Use document IDs from source context or prior search results; otherwise keep the document requirement in query so Knowhere can locate it.", + ) + } + const response = await input.searchSources(request) + for (const result of response.results) { + if (result.source.documentId) knownDocumentIds.add(result.source.documentId) + } + for (const ref of response.referencedChunks) { + if (ref.documentId) knownDocumentIds.add(ref.documentId) + } + return response }, } }, } as const - -async function listVisibleDocuments( - input: NotebookKnowhereToolsInput, -): Promise { - const excludedSourceIds = new Set(input.excludedSourceIds) - const excludedDocumentIds = new Set( - excludeDocuments(input.sources, input.excludedSourceIds) - .excludeDocumentIds ?? [], - ) - const localDocuments = input.sources - .filter( - (source): source is Source & { readonly knowhereDocumentId: string } => - source.status === "ready" && - Boolean(source.knowhereDocumentId) && - !excludedSourceIds.has(source.id) && - !excludedDocumentIds.has(source.knowhereDocumentId ?? ""), - ) - .map((source): KnowhereDocumentSummary => ({ - documentId: source.knowhereDocumentId, - revisionKey: source.knowhereJobId ?? undefined, - namespace: input.namespace, - sourceFileName: source.title, - title: source.title, - status: source.status, - })) - - const remoteDocuments = await listVisibleRemoteDocuments({ - input, - localDocuments, - excludedDocumentIds, - }) - - return [...localDocuments, ...remoteDocuments] -} - -async function listVisibleRemoteDocuments(input: { - readonly input: NotebookKnowhereToolsInput - readonly localDocuments: readonly KnowhereDocumentSummary[] - readonly excludedDocumentIds: ReadonlySet -}): Promise { - if (!input.input.remoteDocumentClient) return [] - - const localDocumentIds = new Set( - input.localDocuments.flatMap((document): string[] => - document.documentId ? [document.documentId] : [], - ), - ) - const documents = await Effect.runPromise( - listRemoteLibraryDocuments({ - workspace: { namespace: input.input.namespace }, - client: input.input.remoteDocumentClient, - localSources: input.input.sources, - }), - ) - - return documents - .filter( - (document) => - isNotebookVisibleRemoteDocument(document) && - document.status === "ready" && - !localDocumentIds.has(document.documentId) && - !input.excludedDocumentIds.has(document.documentId), - ) - .map(toRemoteDocumentSummary) -} - -function toRemoteDocumentSummary( - document: RemoteLibraryDocument, -): KnowhereDocumentSummary { - return { - documentId: document.documentId, - revisionKey: document.revisionKey, - namespace: document.namespace, - sourceFileName: - document.sourceFileName ?? document.title ?? document.documentId, - title: document.title, - status: document.status, - } -} - -function requireKnowledge(knowledge: Knowledge | undefined): Knowledge { - if (knowledge) return knowledge - throw new Error("Knowhere parsed-document reads are not configured.") -} diff --git a/src/domains/chat/media-assets.test.ts b/src/domains/chat/media-assets.test.ts index 470255df..c4ee72f5 100644 --- a/src/domains/chat/media-assets.test.ts +++ b/src/domains/chat/media-assets.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest" import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" import { + dedupeMediaCitationResults, enrichRetrievalResultsWithAssetUrls, formatRetrievedMediaAssetContext, isImageAssetUrl, @@ -9,6 +10,14 @@ import { } from "./media-assets" import type { Source } from "@/infrastructure/db/schema" +vi.mock("@/lib/logger", () => ({ + logger: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, +})) + describe("chat media assets", () => { it("enriches retrieved image chunks from Notebook parsed asset URLs", async () => { const hardenChatAssetUrl = vi @@ -103,8 +112,6 @@ describe("chat media assets", () => { }), ], hardenChatAssetUrl, - evidenceText: - "[image-6-中华人民共和国居民身份证.jpg]\n[image-7-中国居民身份证.jpg]", }) expect(results).toHaveLength(1) @@ -261,6 +268,22 @@ describe("chat media assets", () => { expect(answer).toBe("{\"name\":\"冯荣洲\",\"status\":\"matched\"}") }) + + it("dedupes results with a missing chunkType instead of throwing", () => { + // Knowhere's chunkType is declared as a required string in the SDK + // type, but real retrieval results can omit it at runtime. + const resultWithoutChunkType = makeRetrievalResult({ + chunkType: undefined as unknown as string, + assetUrl: "https://blob.example/images/launch.jpg", + }) + + expect(() => + dedupeMediaCitationResults([resultWithoutChunkType]), + ).not.toThrow() + + const deduped = dedupeMediaCitationResults([resultWithoutChunkType]) + expect(deduped).toEqual([resultWithoutChunkType]) + }) }) function makeRetrievalResult( diff --git a/src/domains/chat/media-assets.ts b/src/domains/chat/media-assets.ts index bafaae88..09c9cd07 100644 --- a/src/domains/chat/media-assets.ts +++ b/src/domains/chat/media-assets.ts @@ -1,7 +1,15 @@ import type { RetrievalResult } from "@ontos-ai/knowhere-sdk" +import { logger } from "@/lib/logger" import type { Source } from "@/infrastructure/db/schema" +// Knowhere's chunkType is declared as a required string in the SDK type, +// but real retrieval results can omit it. Normalize defensively instead of +// calling .toLowerCase() on a value that may be undefined at runtime. +function normalizeChunkType(chunkType: string): string { + return typeof chunkType === "string" ? chunkType.toLowerCase() : "" +} + const retrievedMediaAssetLimit = 6 const imageExtensions = [".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg"] as const const internalMetadataKeys = new Set([ @@ -27,7 +35,6 @@ export type RetrievalResultAssetInput = { readonly results: readonly RetrievalResult[] readonly sources: readonly Source[] readonly hardenChatAssetUrl?: HardenChatAssetUrl - readonly evidenceText?: string } export async function enrichRetrievalResultsWithAssetUrls({ @@ -80,7 +87,7 @@ export function dedupeMediaCitationResults( const existingResult = dedupedResults[existingIndex] const existingAssetUrl = getTrimmedString(existingResult?.assetUrl) - if ( + const keepCurrent = existingResult && existingAssetUrl && compareMediaCitationResult( @@ -89,7 +96,16 @@ export function dedupeMediaCitationResults( assetUrl, existingAssetUrl, ) > 0 - ) { + logger.info("chat-agent: merged duplicate media citation", { + assetKey, + keptChunkId: keepCurrent + ? (result.chunkId ?? null) + : (existingResult?.chunkId ?? null), + droppedChunkId: keepCurrent + ? (existingResult?.chunkId ?? null) + : (result.chunkId ?? null), + }) + if (keepCurrent) { dedupedResults[existingIndex] = result } } @@ -162,7 +178,7 @@ function getMediaCitationResultScore( result: RetrievalResult, assetUrl: string, ): number { - const chunkType = result.chunkType.toLowerCase() + const chunkType = normalizeChunkType(result.chunkType) const isImageAsset = isImageAssetUrl(assetUrl) const isTableAsset = chunkType === "table" const source = result.source @@ -235,7 +251,7 @@ async function addAssetCitationResults( source: Source, hardenChatAssetUrl: HardenChatAssetUrl, ): Promise { - if (result.chunkType.toLowerCase() === "page") return [result] + if (normalizeChunkType(result.chunkType) === "page") return [result] const existingAssetUrl = getTrimmedString(result.assetUrl) if (existingAssetUrl && isNotebookOwnedAssetUrl(existingAssetUrl)) return [result] @@ -434,7 +450,7 @@ function isRenderableMediaAsset( result: RetrievalResult, assetUrl: string, ): boolean { - const chunkType = result.chunkType.toLowerCase() + const chunkType = normalizeChunkType(result.chunkType) return chunkType === "image" || chunkType === "table" || isImageAssetUrl(assetUrl) } diff --git a/src/domains/chat/page-citation-assets.test.ts b/src/domains/chat/page-citation-assets.test.ts index 5b162641..06f679ed 100644 --- a/src/domains/chat/page-citation-assets.test.ts +++ b/src/domains/chat/page-citation-assets.test.ts @@ -203,6 +203,31 @@ describe("enrichRetrievalResultsWithPageCitationAssetUrls", () => { expect(result?.pageCitationAssetUrl).toBeUndefined() expect(result?.pageCitationPageNumber).toBe(4) }) + + it("does not throw when a result is missing chunkType", async () => { + // Knowhere's chunkType is declared as a required string in the SDK + // type, but real retrieval results can omit it at runtime. + const resultWithoutChunkType = makeRetrievalResult({ + chunkType: undefined as unknown as string, + metadata: { + pageNums: [4], + pageAssets: [ + { + pageNum: 4, + artifactRef: "page_citation_assets/page-4.png", + assetUrl: "https://assets.example/pages/page-4.png", + }, + ], + }, + }) + + const [result] = await enrichRetrievalResultsWithPageCitationAssetUrls({ + results: [resultWithoutChunkType], + sources: [makeSource()], + }) + + expect(result?.pageCitationAssetUrl).toBeUndefined() + }) }) function makeRetrievalResult( diff --git a/src/domains/chat/page-citation-assets.ts b/src/domains/chat/page-citation-assets.ts index 3007f60d..07e93197 100644 --- a/src/domains/chat/page-citation-assets.ts +++ b/src/domains/chat/page-citation-assets.ts @@ -122,7 +122,12 @@ async function getStoredPageCitationAssetUrl(input: { } function isPageResult(result: RetrievalResult): boolean { - return result.chunkType.toLowerCase() === "page" + // Knowhere's chunkType is declared as a required string in the SDK type, + // but real retrieval results can omit it. Normalize defensively instead + // of calling .toLowerCase() on a value that may be undefined at runtime. + return typeof result.chunkType === "string" + ? result.chunkType.toLowerCase() === "page" + : false } function getDirectPageCitationAsset( diff --git a/src/domains/chat/prompt.ts b/src/domains/chat/prompt.ts index a4ae326d..4d24c99a 100644 --- a/src/domains/chat/prompt.ts +++ b/src/domains/chat/prompt.ts @@ -16,6 +16,7 @@ import type { ChatHistoryMessage, SearchSources, } from "./contracts" +import { mementoMemoryTools } from "@/integrations/memento/memory-tools" import { notebookKnowhereTools } from "./knowhere-tools" const RECENT_CONTEXT_MESSAGE_LIMIT = 8 @@ -23,6 +24,7 @@ const CONTEXT_CONTENT_CHAR_LIMIT = 900 const SOURCE_CONTEXT_LIMIT = 12 type GenerateAgenticOutputManifestInput = { + workspaceId: string question: string messages: readonly ChatHistoryMessage[] sources: readonly Source[] @@ -60,9 +62,13 @@ export const generateAgenticOutputManifestEffect = ( turn, knowhereTools: input.knowhereTools ?? - notebookKnowhereTools.createSearchOnlyRuntime({ + notebookKnowhereTools.createRuntime({ searchSources: input.searchSources, + sources: input.sources, }), + memoryTools: mementoMemoryTools.createRuntime({ + workspaceId: input.workspaceId, + }), ...(input.inspectImages ? { inspectImages: input.inspectImages } : {}), }), ) diff --git a/src/domains/chat/retrieval.ts b/src/domains/chat/retrieval.ts index 4d1103fa..e5391136 100644 --- a/src/domains/chat/retrieval.ts +++ b/src/domains/chat/retrieval.ts @@ -1,4 +1,4 @@ -import type { RetrievalQueryParams } from "@ontos-ai/knowhere-sdk" +import type { KnowhereSearchRequest } from "@/agent-harness/types" import type { Source } from "@/infrastructure/db/schema" import { decodeRemoteSourceId } from "@/domains/sources/remote-library" @@ -21,10 +21,11 @@ export function normalizeRetrievalQuery(value: string, fallback: string): string return normalized.slice(0, RETRIEVAL_QUERY_CHAR_LIMIT) } -export function excludeDocuments( +export function getRetrievalDocumentScope( sources: readonly Source[], excludedSourceIds: readonly string[], -): Pick { + request: Pick, +): Pick { const excluded = new Set(excludedSourceIds) const localDocumentIds = sources .filter((source) => excluded.has(source.id)) @@ -34,10 +35,19 @@ export function excludeDocuments( .map((sourceId) => decodeRemoteSourceId(sourceId)?.documentId) .filter((documentId): documentId is string => Boolean(documentId)) const documentIds = Array.from( - new Set([...localDocumentIds, ...remoteDocumentIds]), + new Set([ + ...localDocumentIds, + ...remoteDocumentIds, + ...(request.excludeDocumentIds ?? []), + ]), ) - return documentIds.length > 0 ? { excludeDocumentIds: documentIds } : {} + return { + ...(request.includeDocumentIds !== undefined + ? { includeDocumentIds: [...new Set(request.includeDocumentIds)] } + : {}), + ...(documentIds.length > 0 ? { excludeDocumentIds: documentIds } : {}), + } } function stripWrappingQuotes(value: string): string { diff --git a/src/domains/chat/route-answer.ts b/src/domains/chat/route-answer.ts index cc579b05..34cc4d7d 100644 --- a/src/domains/chat/route-answer.ts +++ b/src/domains/chat/route-answer.ts @@ -1,9 +1,6 @@ import { Cause, Effect, Either, Option } from "effect" -import { - generateAgenticOutputManifest, - parseChatRequestBody, -} from "@/domains/chat" +import { parseChatRequestBody } from "@/domains/chat" import type { ImageInspectionAsset, ImageInspectionRequest, @@ -11,16 +8,15 @@ import type { ImageInspectionSkippedAsset, InspectImages, } from "@/agent-harness" +import { commitAgenticChatTurn } from "@/domains/chat/commit-turn" import { normalizeImageInspectionHighlights } from "@/agent-harness/image-highlights" import { generateImageInspectionModelResult } from "@/domains/chat/image-inspection-model" import { hardenChatMediaAssetUrls } from "@/domains/chat/media-asset-hardening" import { - handleChatTurn, type ChatTurnError, type ChatTurnValue, } from "@/domains/chat/service" import { chatTurnPersistence } from "@/domains/chat/chat-turn-persistence" -import { triggerMemoryExtraction } from "@/domains/memory/extract-trigger" import { startBackgroundReconciliation } from "@/domains/sources/background-reconcile" import { BlobParsedDocumentStorage } from "@/domains/sources/parsed-document-blob-storage" import { sourceWorkflowRuntime } from "@/domains/sources/workflow-runtime" @@ -140,7 +136,7 @@ const answerChatEffect = (input: AnswerChatInput) => const result: Either.Either = yield* Effect.tryPromise(() => - handleChatTurn({ + commitAgenticChatTurn({ workspace, sources, question: body.value.question, @@ -149,8 +145,6 @@ const answerChatEffect = (input: AnswerChatInput) => excludedSourceIds: body.value.excludedSourceIds, retrieval: client.retrieval, knowledge: knowhereResources.knowledge, - remoteDocumentClient: client, - generateAnswer: generateAgenticOutputManifest, hardenChatAssetUrl, hardenMediaAssetUrls: ({ results, artifacts }) => hardenChatMediaAssetUrls({ @@ -197,17 +191,8 @@ const answerChatEffect = (input: AnswerChatInput) => return Either.match(result, { onLeft: (error): RouteResponse => routeResult.error(error.status, error.message), - onRight: (value): RouteResponse => { - // Fire-and-forget: extract fluid memory from this turn without - // blocking the chat response. - void triggerMemoryExtraction({ - workspaceId: workspace.id, - threadId: value.threadId, - userMessageId: value.messages[0].id, - assistantMessageId: value.messages[1].id, - }) - return routeResult.ok(value) - }, + onRight: (value): RouteResponse => + routeResult.ok(value), }) }) diff --git a/src/domains/chat/route-service.test.ts b/src/domains/chat/route-service.test.ts index 02eceb9b..d27b32dd 100644 --- a/src/domains/chat/route-service.test.ts +++ b/src/domains/chat/route-service.test.ts @@ -25,7 +25,8 @@ const mocks = vi.hoisted(() => ({ parsedStorageWriteAsset: vi.fn(), softDeleteChatThread: vi.fn(), startBackgroundReconciliation: vi.fn(), - triggerMemoryExtraction: vi.fn(), + captureMemoryTurn: vi.fn(), + recordActivations: vi.fn(), })) vi.mock("ai", async (importOriginal) => { @@ -63,8 +64,9 @@ vi.mock("@/domains/sources/background-reconcile", () => ({ startBackgroundReconciliation: mocks.startBackgroundReconciliation, })) -vi.mock("@/domains/memory/extract-trigger", () => ({ - triggerMemoryExtraction: mocks.triggerMemoryExtraction, +vi.mock("@/integrations/memento/client", () => ({ + captureMemoryTurn: mocks.captureMemoryTurn, + recordActivations: mocks.recordActivations, })) vi.mock("@/domains/sources/workflow-runtime", () => ({ @@ -191,7 +193,7 @@ describe("chat route services", () => { useAgentic: true, excludedSourceIds: ["source_skipped"], retrieval: client.retrieval, - generateAnswer: mocks.generateAgenticOutputManifest, + generateAnswer: expect.any(Function), hardenChatAssetUrl: expect.any(Function), repository: expect.objectContaining({ appendMessageToThread: expect.any(Function), @@ -853,11 +855,12 @@ describe("chat route services", () => { }) expect(result.status).toBe(200) - expect(mocks.triggerMemoryExtraction).toHaveBeenCalledWith({ + expect(mocks.captureMemoryTurn).toHaveBeenCalledWith({ workspaceId: workspace.id, - threadId: "thread_1", - userMessageId: "message_user", - assistantMessageId: "message_assistant", + sourceMessageId: "message_assistant", + userText: "Summarize it", + assistantText: "Summary", + referencedDocumentIds: [], }) expect(mocks.startBackgroundReconciliation).toHaveBeenCalledWith( workspace.id, diff --git a/src/domains/chat/service.test.ts b/src/domains/chat/service.test.ts index b0652519..4aa1a5a0 100644 --- a/src/domains/chat/service.test.ts +++ b/src/domains/chat/service.test.ts @@ -54,7 +54,7 @@ describe("handleChatTurn", () => { }); } expect(retrieval.query).toHaveBeenCalledWith({ - namespace: "notebook-namespace", + namespace: "default", query: "What does the document say?", topK: 8, useAgentic: true, @@ -83,6 +83,31 @@ describe("handleChatTurn", () => { }); }); + it("rejects a turn with no local sources without calling retrieval", async () => { + const retrieval = { query: vi.fn() }; + const repository = makeRepository(); + + const result = await handleChatTurn({ + workspace: makeWorkspace(), + sources: [], + question: "What does the document say?", + excludedSourceIds: [], + retrieval, + generateAnswer: vi.fn(), + repository, + }); + + expect(Either.isLeft(result)).toBe(true); + if (Either.isLeft(result)) { + expect(result.left).toMatchObject({ + status: 409, + message: "Upload and process a document before asking questions.", + }); + } + expect(retrieval.query).not.toHaveBeenCalled(); + expect(repository.appendMessageToThread).not.toHaveBeenCalled(); + }); + it("rejects chat before any source is ready without calling retrieval", async () => { const retrieval = { query: vi.fn() }; const repository = makeRepository(); @@ -220,7 +245,7 @@ describe("handleChatTurn", () => { knowhereTools: expect.any(Object), }); expect(retrieval.query).toHaveBeenCalledWith({ - namespace: "notebook-namespace", + namespace: "default", query: "Tesla Q4 2025 Update energy generation and storage deployments", topK: 8, useAgentic: true, @@ -334,6 +359,7 @@ function makeHarnessRunResult(text: string): HarnessRunResult { manifest: { text, citations: [], + memoryCitations: [], artifacts: [], unresolved: [], }, diff --git a/src/domains/chat/service.ts b/src/domains/chat/service.ts index 23bfa68d..8139a42f 100644 --- a/src/domains/chat/service.ts +++ b/src/domains/chat/service.ts @@ -9,7 +9,7 @@ import { } from "." import { toChatMessageView } from "./view" import type { ChatMessage, ChatThread, Source, Workspace } from "@/infrastructure/db/schema" -import { getCompatibleNamespaces } from "@/domains/sources/namespace" +import { sharedLibraryNamespace } from "@/domains/sources/namespace" import type { ChatArtifactView, ChatCitationView, @@ -68,7 +68,6 @@ type ChatTurnInput = { excludedSourceIds: readonly string[] retrieval: RetrievalClient knowledge?: AnswerQuestionInput["knowledge"] - remoteDocumentClient?: AnswerQuestionInput["remoteDocumentClient"] generateAnswer: GenerateAnswer hardenChatAssetUrl?: AnswerQuestionInput["hardenChatAssetUrl"] hardenMediaAssetUrls?: AnswerQuestionInput["hardenMediaAssetUrls"] @@ -125,13 +124,12 @@ export const handleChatTurnEffect = (input: ChatTurnInput) => const answer = yield* answerQuestionWithRetrieval({ question: input.question, namespace: input.workspace.namespace, - namespaces: getCompatibleNamespaces(input.workspace), + namespaces: [sharedLibraryNamespace], sources: readySources, useAgentic: input.useAgentic ?? true, excludedSourceIds: input.excludedSourceIds, retrieval: input.retrieval, knowledge: input.knowledge, - remoteDocumentClient: input.remoteDocumentClient, generateAnswer: input.generateAnswer, hardenChatAssetUrl: input.hardenChatAssetUrl, hardenMediaAssetUrls: input.hardenMediaAssetUrls, diff --git a/src/domains/chat/types.ts b/src/domains/chat/types.ts index 1d6f88f2..e10644f6 100644 --- a/src/domains/chat/types.ts +++ b/src/domains/chat/types.ts @@ -5,6 +5,7 @@ export type RetrievalResultView = { readonly content: string readonly chunkType: string readonly score: number | null + readonly chunkId?: string readonly assetUrl?: string readonly pageCitationAssetUrl?: string readonly pageCitationPageNumber?: number diff --git a/src/domains/memory/extract-trigger.ts b/src/domains/memory/extract-trigger.ts deleted file mode 100644 index 35b64675..00000000 --- a/src/domains/memory/extract-trigger.ts +++ /dev/null @@ -1,43 +0,0 @@ -import "server-only" - -import { Client } from "@upstash/workflow" - -import type { MemoryExtractPayload } from "./extract-workflow" -import { logger } from "@/lib/logger" - -function resolveBaseURL(): string { - return process.env.NOTEBOOK_PUBLIC_URL ?? "http://localhost:3000" -} - -/** - * Fire-and-forget trigger for the post-turn fluid-memory extraction - * workflow. Each turn is uniquely keyed by its message ids, so unlike the - * source reconcile trigger no cooldown/dedup guard is needed; QStash - * retries cover transient delivery failures. - */ -export async function triggerMemoryExtraction( - payload: MemoryExtractPayload, -): Promise { - const token = process.env.QSTASH_TOKEN - if (!token) { - logger.warn("memory: skipping extraction — QSTASH_TOKEN not set", { - workspaceId: payload.workspaceId, - assistantMessageId: payload.assistantMessageId, - }) - return - } - - try { - await new Client({ token }).trigger({ - url: `${resolveBaseURL()}/api/memory/extract`, - body: payload, - retries: 3, - }) - } catch (error) { - logger.error("memory: failed to trigger extraction workflow", { - workspaceId: payload.workspaceId, - assistantMessageId: payload.assistantMessageId, - message: error instanceof Error ? error.message : String(error), - }) - } -} diff --git a/src/domains/memory/extract-workflow.ts b/src/domains/memory/extract-workflow.ts deleted file mode 100644 index aa6b5ee3..00000000 --- a/src/domains/memory/extract-workflow.ts +++ /dev/null @@ -1,169 +0,0 @@ -import "server-only" - -import type { WorkflowContext } from "@upstash/workflow" - -import { extractMemoryOperations } from "./extraction-model" -import { - summarizePayloadForContext, - type ExistingMemoryContextItem, -} from "./prompts" -import { resolveMemoryOperations } from "./resolve-operations" -import { tokenizeMemoryText } from "./search-index" -import { memoryService } from "./service" -import { fluidMemoryKinds, isFluidMemoryKind } from "./types" -import { chatThreadService } from "@/domains/chat/thread-service" -import { logger } from "@/lib/logger" - -export type MemoryExtractPayload = { - readonly workspaceId: string - readonly threadId: string - readonly userMessageId: string - readonly assistantMessageId: string -} - -type MemoryExtractWorkflowContext = Pick< - WorkflowContext, - "run" -> - -/** Prompt-context item that also carries status/payload for resolution. */ -type MemoryWorkflowItem = ExistingMemoryContextItem & { - readonly status: string - readonly payload: unknown -} - -/** Per-kind cap on lexical neighbors fed into the merge-decision prompt. */ -const DEDUP_CANDIDATES_PER_KIND = 8 - -export function normalizeMemoryExtractPayload( - raw: unknown, -): MemoryExtractPayload | null { - if (!raw || typeof raw !== "object") return null - const record = raw as Record - const workspaceId = getNonEmptyString(record.workspaceId) - const threadId = getNonEmptyString(record.threadId) - const userMessageId = getNonEmptyString(record.userMessageId) - const assistantMessageId = getNonEmptyString(record.assistantMessageId) - if (!workspaceId || !threadId || !userMessageId || !assistantMessageId) { - return null - } - return { workspaceId, threadId, userMessageId, assistantMessageId } -} - -export async function runMemoryExtractWorkflow(input: { - readonly context: MemoryExtractWorkflowContext - readonly payload: MemoryExtractPayload -}): Promise { - const { context, payload } = input - - const turn = await context.run("load-turn", async () => { - const messages = await chatThreadService.listMessages( - payload.workspaceId, - payload.threadId, - ) - const userMessage = messages?.find( - (message) => message.id === payload.userMessageId, - ) - const assistantMessage = messages?.find( - (message) => message.id === payload.assistantMessageId, - ) - if (!userMessage || !assistantMessage) return null - return { - userText: userMessage.content, - assistantText: assistantMessage.content, - referencedDocumentIds: collectCitationDocumentIds( - assistantMessage.citations, - ), - } - }) - if (!turn) { - logger.warn("memory: extract skipped — turn messages not found", { - workspaceId: payload.workspaceId, - threadId: payload.threadId, - }) - return - } - - const existingItems = await context.run("retrieve-candidates", async () => { - const queryTokens = tokenizeMemoryText(turn.userText).map( - (entry) => entry.token, - ) - if (queryTokens.length === 0) return [] - - const byId = new Map() - for (const kind of fluidMemoryKinds) { - const items = await memoryService.findDedupCandidates( - payload.workspaceId, - kind, - queryTokens, - DEDUP_CANDIDATES_PER_KIND, - ) - for (const item of items) { - if (!isFluidMemoryKind(item.kind) || byId.has(item.id)) continue - byId.set(item.id, { - id: item.id, - kind: item.kind, - status: item.status, - payload: item.payload, - abstractL0: item.abstractL0, - payloadSummary: summarizePayloadForContext(item.kind, item.payload), - }) - } - } - return [...byId.values()] - }) - - const operations = await context.run("extract-operations", () => - extractMemoryOperations({ - workspaceId: payload.workspaceId, - userText: turn.userText, - assistantText: turn.assistantText, - referencedDocumentIds: turn.referencedDocumentIds, - existingItems, - }), - ) - if (!operations) return - - const applied = await context.run("apply-operations", async () => { - const resolved = resolveMemoryOperations({ - operations, - existingItems, - referencedDocumentIds: turn.referencedDocumentIds, - }) - if (resolved.length === 0) return null - return memoryService.applyOperations( - payload.workspaceId, - payload.assistantMessageId, - resolved, - ) - }) - - logger.info("memory: extract workflow finished", { - workspaceId: payload.workspaceId, - threadId: payload.threadId, - assistantMessageId: payload.assistantMessageId, - candidateCount: existingItems.length, - appliedOperations: applied?.map((operation) => operation.op) ?? [], - }) -} - -function collectCitationDocumentIds(citations: unknown): string[] { - if (!Array.isArray(citations)) return [] - const ids = new Set() - for (const citation of citations) { - if (!citation || typeof citation !== "object") continue - const source = (citation as Record).source - if (!source || typeof source !== "object") continue - const documentId = (source as Record).documentId - if (typeof documentId === "string" && documentId.length > 0) { - ids.add(documentId) - } - } - return [...ids] -} - -function getNonEmptyString(value: unknown): string | null { - return typeof value === "string" && value.trim().length > 0 - ? value - : null -} diff --git a/src/domains/memory/extraction-model.ts b/src/domains/memory/extraction-model.ts deleted file mode 100644 index fa490115..00000000 --- a/src/domains/memory/extraction-model.ts +++ /dev/null @@ -1,51 +0,0 @@ -import "server-only" - -import { generateObject } from "ai" - -import { - buildMemoryExtractionPrompt, - memoryOperationsSchema, - type ExistingMemoryContextItem, - type MemoryOperations, -} from "./prompts" -import { CHAT_MODEL } from "@/lib/ai" -import { summarizeUnknownError } from "@/lib/format-log-value" -import { logger } from "@/lib/logger" - -const MEMORY_EXTRACTION_MODEL = process.env.MEMORY_EXTRACTION_MODEL ?? CHAT_MODEL - -/** - * One structured-output call: turn + existing active memories in, typed - * operations out. Best-effort by design — this runs as a background job, - * so a model failure skips the turn (logged) instead of degrading through - * fallbacks; the insight typically resurfaces in a later turn. - */ -export async function extractMemoryOperations(input: { - readonly workspaceId: string - readonly userText: string - readonly assistantText: string - readonly referencedDocumentIds: readonly string[] - readonly existingItems: readonly ExistingMemoryContextItem[] -}): Promise { - try { - const response = await generateObject({ - model: MEMORY_EXTRACTION_MODEL, - schema: memoryOperationsSchema, - messages: [ - { - role: "user", - content: buildMemoryExtractionPrompt(input), - }, - ], - }) - return response.object - } catch (error) { - logger.warn("memory: extraction model call failed; skipping turn", { - workspaceId: input.workspaceId, - model: MEMORY_EXTRACTION_MODEL, - existingItemCount: input.existingItems.length, - error: summarizeUnknownError(error), - }) - return null - } -} diff --git a/src/domains/memory/prompts.test.ts b/src/domains/memory/prompts.test.ts deleted file mode 100644 index 2ae15ff2..00000000 --- a/src/domains/memory/prompts.test.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { buildMemoryExtractionPrompt } from "./prompts" - -describe("buildMemoryExtractionPrompt", () => { - const prompt = buildMemoryExtractionPrompt({ - userText: "毛利率是核心。", - assistantText: "明白。", - referencedDocumentIds: ["doc-1"], - existingItems: [], - }) - - it("keeps main instructions domain-agnostic", () => { - const main = prompt.slice(0, prompt.indexOf("## Illustrative examples")) - expect(main).not.toMatch(/gross margin|市盈率|\bPE\b|Fed|美联储|NVIDIA|英伟达/i) - expect(main).toContain("Write every free-text value") - expect(main).toMatch(/same language the\s+USER wrote/) - }) - - it("keeps illustrative examples in a separate section", () => { - expect(prompt).toContain("## Illustrative examples (finance vertical") - expect(prompt).toContain("not exhaustive, not required vocabulary") - const examples = prompt.slice( - prompt.indexOf("## Illustrative examples"), - prompt.indexOf("## Output JSON schema"), - ) - expect(examples).toContain("finance vertical") - expect(examples).toContain("do not force the conversation into this domain") - }) - - it("still injects turn context after the fixed blocks", () => { - expect(prompt).toContain("[user]\n毛利率是核心。") - expect(prompt).toContain("doc-1") - }) -}) diff --git a/src/domains/memory/prompts.ts b/src/domains/memory/prompts.ts deleted file mode 100644 index e2fbeb0c..00000000 --- a/src/domains/memory/prompts.ts +++ /dev/null @@ -1,285 +0,0 @@ -import { z } from "zod" - -import { - decisionRulePayloadSchema, - entityOfInterestPayloadSchema, - indicatorPreferencePayloadSchema, - stancePayloadSchema, - type FluidMemoryKind, -} from "./types" - -/** - * LLM contract for fluid-memory extraction. - * - * Design borrowed from OpenViking's session-commit extraction (schema-driven - * typed operations, prefetch-then-decide), reimplemented as a single - * structured-output call: the model sees the turn plus lexically retrieved - * dedup candidates and directly outputs per-kind operations - * (create / skip / merge / deprecate), mirroring OpenViking's generated - * operations model without its ReAct tool loop. - */ - -const decisionSchema = z.object({ - op: z.enum(["create", "skip", "merge", "deprecate"]), - targetItemId: z.preprocess( - (value) => (value === null ? undefined : value), - z - .string() - .optional() - .describe( - "Required for merge/deprecate: the id of the existing memory item this operation targets. Omit for create/skip.", - ), - ), - reason: z.preprocess( - (value) => (value === null ? undefined : value), - z - .string() - .optional() - .describe("Short justification, especially for skip/merge/deprecate."), - ), -}) - -const memorySidecarFields = { - abstractL0: z - .string() - .min(1) - .describe("One line, <= 30 words: the essence of this insight."), - overviewL1: z - .string() - .min(1) - .describe("2-3 sentences: what it means and when it applies."), - confidence: z - .number() - .min(0) - .max(1) - .describe("How explicitly the user stated this (1 = explicit)."), - decision: decisionSchema, -} - -const stanceEntrySchema = z.preprocess((value) => { - if (!value || typeof value !== "object" || Array.isArray(value)) return value - const record = value as Record - // Models sometimes emit "name" for a stance; the contract field is "statement". - if ( - (typeof record.statement !== "string" || record.statement.length === 0) && - typeof record.name === "string" && - record.name.length > 0 - ) { - const { name, ...rest } = record - return { ...rest, statement: name } - } - return value -}, stancePayloadSchema.extend(memorySidecarFields)) - -const entityEntrySchema = z.preprocess((value) => { - if (!value || typeof value !== "object" || Array.isArray(value)) return value - const record = value as Record - // Keep provenance reason required; if the model omitted it, fall back to L0. - if ( - (typeof record.reason !== "string" || record.reason.length === 0) && - typeof record.abstractL0 === "string" && - record.abstractL0.length > 0 - ) { - return { ...record, reason: record.abstractL0 } - } - return value -}, entityOfInterestPayloadSchema.extend(memorySidecarFields)) - -export const memoryOperationsSchema = z.object({ - indicatorPrefs: z - .array(indicatorPreferencePayloadSchema.extend(memorySidecarFields)) - .default([]), - stances: z.array(stanceEntrySchema).default([]), - decisionRules: z - .array(decisionRulePayloadSchema.extend(memorySidecarFields)) - .default([]), - entities: z.array(entityEntrySchema).default([]), -}) - -export type MemoryOperations = z.infer - -export type ExistingMemoryContextItem = { - readonly id: string - readonly kind: FluidMemoryKind - readonly abstractL0: string - readonly payloadSummary: string -} - -/** Structural output shape only — no domain content. */ -const OUTPUT_SCHEMA_BLOCK = `{ - "indicatorPrefs": [{ - "name": "string", - "aliases": ["string"], - "definition": "string", - "polarity": "higher_better|lower_better|context", - "importance": "core|secondary", - "formulaHint": "string (optional — omit if none)", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }], - "stances": [{ - "statement": "string (the stance text; do not use a name field)", - "scope": "string", - "rationale": "string", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }], - "decisionRules": [{ - "when": "string", - "then": "string", - "priority": "high|medium|low", - "rationale": "string", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }], - "entities": [{ - "name": "string", - "ticker": "string optional", - "aliases": ["string"], - "knowhereDocumentIds": ["only ids listed under REFERENCED DOCUMENT IDS"], - "reason": "string", - "abstractL0": "string", - "overviewL1": "string", - "confidence": 0.0, - "decision": { "op": "create|skip|merge|deprecate", "targetItemId": "id for merge/deprecate only", "reason": "string optional" } - }] -}` - -/** - * Illustrative only — kept separate from the main instructions so the model - * does not treat these domain phrases as required vocabulary. - * Finance is the first vertical; add other industry blocks here later if needed. - */ -const ILLUSTRATIVE_EXAMPLES_BLOCK = `## Illustrative examples (finance vertical — not exhaustive, not required vocabulary) - -These show shape and judgement only. Extract whatever the user actually said; -do not force the conversation into this domain or these metric names. - -- indicatorPref: user says a named metric they repeatedly use to judge quality - (shape: name + short definition + polarity + importance). Same idea applies - outside finance (any recurring evaluation metric). -- stance: user states a durable judgement frame that changes how evidence is - weighted (e.g. long-horizon vs short-horizon). -- decisionRule: user states a reusable when → then discipline over their metrics. -- entity: user says they actively track a named company/issuer and why. -- skip: a one-off factual question about a page/number in a document, small talk, - or an assistant suggestion the user did not endorse.` - -/** Domain-agnostic extraction instructions. */ -const MAIN_INSTRUCTIONS_BLOCK = `You maintain a user's FLUID MEMORY: durable insights about how this user thinks, extracted from their conversation with an AI analyst. - -Document facts live elsewhere (crystal memory). Never extract document facts, retrieved numbers, or page content as fluid memory. - -## What to extract - -Extract ONLY these four kinds, and ONLY when the turn gives real evidence from the USER: - -- indicatorPrefs — recurring metrics or criteria the user uses to evaluate things. - Fields: name, aliases, definition, polarity (higher_better | lower_better | context), - importance (core | secondary), optional formulaHint. -- stances — durable positions that shape how the user weighs evidence. - Fields: statement (required; do not invent a "name" field), scope, rationale. -- decisionRules — reusable when → then disciplines the user stated or clearly endorsed. - Fields: when, then, priority (high | medium | low), rationale. -- entities — named subjects the user is actively tracking. - Fields: name, optional ticker, aliases, reason (required), knowhereDocumentIds - (only from REFERENCED DOCUMENT IDS below; never invent ids). - -## Language - -- Keep this instruction set and enum/field names in English. -- Write every free-text value (name, definition, statement, when/then, reason, - abstractL0, overviewL1, aliases the user used, etc.) in the same language the - USER wrote in this turn. Do not translate the user's terms into English unless - the user themselves used English. - -## Decision rules - -- Extract only durable, reusable insights about the USER. -- Skip one-off questions, document facts, small talk, and assistant claims the user did not endorse. -- For every candidate, choose exactly one op against EXISTING MEMORIES: - - create — genuinely new - - skip — already covered, or too weak/ephemeral - - merge — same insight refined; emit the full merged fields and set targetItemId - - deprecate — user explicitly reversed a stored item; set targetItemId -- Be conservative: prefer create over merge when overlap is only partial; deprecate only on clear contradiction. -- Prefer one record per insight. If a preference already encodes how a metric should be read, do not also invent a near-duplicate decisionRule unless the user stated an explicit when → then action. -- abstractL0: one line. overviewL1: 2–3 sentences. confidence=1 only when the user stated it explicitly. -- Omit optional fields instead of setting them to null. -- If nothing is worth remembering, return all four arrays empty.` - -export function buildMemoryExtractionPrompt(input: { - readonly userText: string - readonly assistantText: string - readonly referencedDocumentIds: readonly string[] - readonly existingItems: readonly ExistingMemoryContextItem[] -}): string { - const existingBlock = - input.existingItems.length === 0 - ? "(no existing memories yet)" - : input.existingItems - .map( - (item) => - `- [${item.kind}] id=${item.id} :: ${item.abstractL0} :: ${item.payloadSummary}`, - ) - .join("\n") - - const documentsBlock = - input.referencedDocumentIds.length === 0 - ? "(no documents referenced in this turn)" - : input.referencedDocumentIds.join(", ") - - return `${MAIN_INSTRUCTIONS_BLOCK} - -${ILLUSTRATIVE_EXAMPLES_BLOCK} - -## Output JSON schema (follow exactly; do not invent fields) - -${OUTPUT_SCHEMA_BLOCK} - -## EXISTING MEMORIES - -${existingBlock} - -## REFERENCED DOCUMENT IDS - -${documentsBlock} - -## CONVERSATION TURN - -[user] -${input.userText} - -[assistant] -${input.assistantText}` -} - -export function summarizePayloadForContext( - kind: FluidMemoryKind, - payload: unknown, -): string { - if (!payload || typeof payload !== "object") return "" - const record = payload as Record - switch (kind) { - case "indicator_pref": - return [record.name, record.definition] - .filter((part) => typeof part === "string" && part.length > 0) - .join(" — ") - case "stance": - return typeof record.statement === "string" ? record.statement : "" - case "decision_rule": - return [record.when, record.then] - .filter((part) => typeof part === "string" && part.length > 0) - .join(" => ") - case "entity_of_interest": - return [record.name, record.ticker] - .filter((part) => typeof part === "string" && part.length > 0) - .join(" ") - } -} diff --git a/src/domains/memory/repository.ts b/src/domains/memory/repository.ts deleted file mode 100644 index f7f6b794..00000000 --- a/src/domains/memory/repository.ts +++ /dev/null @@ -1,254 +0,0 @@ -import "server-only" - -import { and, eq, inArray, sql } from "drizzle-orm" -import { Effect } from "effect" - -import { toDiffOperation, type ResolvedMemoryOperation } from "./resolve-operations" -import { buildMemoryItemTokens } from "./search-index" -import type { - FluidMemoryKind, - FluidMemoryPayload, - MemoryDiffOperation, -} from "./types" -import { DbClient } from "@/infrastructure/db" -import { - fluidMemoryItems, - fluidMemoryTokens, - memoryDiffs, - type FluidMemoryItem, - type NewFluidMemoryToken, -} from "@/infrastructure/db/schema" - -type MemoryRepository = { - readonly findDedupCandidatesEffect: ( - workspaceId: string, - kind: FluidMemoryKind, - tokens: readonly string[], - limit: number, - ) => Effect.Effect - readonly applyOperationsEffect: ( - workspaceId: string, - sourceMessageId: string | null, - operations: readonly ResolvedMemoryOperation[], - ) => Effect.Effect -} - -type RawRowsResult = readonly Row[] | { readonly rows: readonly Row[] } - -/** - * Retrieve the most lexically-similar active items of one kind, ranked by - * idf-weighted token overlap computed entirely in SQL. Common tokens (high - * document frequency within this workspace + kind) are down-weighted so a - * shared rare term outranks several shared filler characters. - * - * Token rows only exist for active items (see schema invariant), so no - * status filter is needed here. - */ -const findDedupCandidatesEffect: MemoryRepository["findDedupCandidatesEffect"] = - (workspaceId, kind, tokens, limit) => - Effect.gen(function* () { - const db = yield* DbClient - if (tokens.length === 0 || limit <= 0) return [] - - const tokenList = sql.join( - tokens.map((token) => sql`${token}`), - sql`, `, - ) - - const scored = yield* Effect.promise(() => - db.execute<{ itemId: string }>(sql` - SELECT t.item_id AS "itemId", SUM(t.frequency::float8 / df.df) AS score - FROM fluid_memory_tokens t - JOIN ( - SELECT token, COUNT(DISTINCT item_id)::float8 AS df - FROM fluid_memory_tokens - WHERE workspace_id = ${workspaceId}::uuid - AND kind = ${kind} - AND token IN (${tokenList}) - GROUP BY token - ) df ON df.token = t.token - WHERE t.workspace_id = ${workspaceId}::uuid - AND t.kind = ${kind} - AND t.token IN (${tokenList}) - GROUP BY t.item_id - ORDER BY score DESC - LIMIT ${limit} - `), - ) - - const orderedIds = getRawRows(scored).map((row) => row.itemId) - if (orderedIds.length === 0) return [] - - const items = yield* Effect.promise(() => - db - .select() - .from(fluidMemoryItems) - .where(inArray(fluidMemoryItems.id, orderedIds)), - ) - const byId = new Map(items.map((item) => [item.id, item] as const)) - return orderedIds.flatMap((id) => { - const item = byId.get(id) - return item ? [item] : [] - }) - }) - -const applyOperationsEffect: MemoryRepository["applyOperationsEffect"] = ( - workspaceId, - sourceMessageId, - operations, -) => - Effect.gen(function* () { - const db = yield* DbClient - return yield* Effect.promise(() => - db.transaction(async (tx) => { - const diffOperations: MemoryDiffOperation[] = [] - - for (const operation of operations) { - switch (operation.op) { - case "create": { - const [inserted] = await tx - .insert(fluidMemoryItems) - .values({ - workspaceId, - kind: operation.kind, - payload: operation.payload, - abstractL0: operation.abstractL0, - overviewL1: operation.overviewL1, - sourceMessageId, - confidence: operation.confidence, - status: "active", - }) - .returning() - if (inserted?.id) { - const tokenRows = tokenRowsFor( - workspaceId, - inserted.id, - operation.kind, - operation.payload, - ) - if (tokenRows.length > 0) { - await tx.insert(fluidMemoryTokens).values(tokenRows) - } - } - diffOperations.push(toDiffOperation(operation, inserted?.id)) - break - } - case "merge": { - const [updated] = await tx - .update(fluidMemoryItems) - .set({ - payload: operation.payload, - abstractL0: operation.abstractL0, - overviewL1: operation.overviewL1, - confidence: operation.confidence, - sourceMessageId, - version: sql`${fluidMemoryItems.version} + 1`, - updatedAt: sql`now()`, - }) - .where( - and( - eq(fluidMemoryItems.id, operation.targetItemId), - eq(fluidMemoryItems.status, "active"), - ), - ) - .returning({ id: fluidMemoryItems.id }) - if (updated) { - await tx - .delete(fluidMemoryTokens) - .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) - const tokenRows = tokenRowsFor( - workspaceId, - operation.targetItemId, - operation.kind, - operation.payload, - ) - if (tokenRows.length > 0) { - await tx.insert(fluidMemoryTokens).values(tokenRows) - } - } - diffOperations.push( - updated - ? toDiffOperation(operation) - : { - op: "skip", - kind: operation.kind, - summary: operation.summary, - reason: "merge target no longer active", - }, - ) - break - } - case "deprecate": { - const [updated] = await tx - .update(fluidMemoryItems) - .set({ - status: "deprecated", - updatedAt: sql`now()`, - }) - .where( - and( - eq(fluidMemoryItems.id, operation.targetItemId), - eq(fluidMemoryItems.status, "active"), - ), - ) - .returning({ id: fluidMemoryItems.id }) - if (updated) { - await tx - .delete(fluidMemoryTokens) - .where(eq(fluidMemoryTokens.itemId, operation.targetItemId)) - } - diffOperations.push( - updated - ? toDiffOperation(operation) - : { - op: "skip", - kind: operation.kind, - summary: operation.summary, - reason: "deprecate target no longer active", - }, - ) - break - } - case "skip": - diffOperations.push(toDiffOperation(operation)) - break - } - } - - if (diffOperations.length > 0) { - await tx.insert(memoryDiffs).values({ - workspaceId, - sourceMessageId, - operations: [...diffOperations], - }) - } - - return diffOperations - }), - ) - }) - -export const memoryRepository: MemoryRepository = { - findDedupCandidatesEffect, - applyOperationsEffect, -} - -function tokenRowsFor( - workspaceId: string, - itemId: string, - kind: FluidMemoryKind, - payload: FluidMemoryPayload, -): NewFluidMemoryToken[] { - return buildMemoryItemTokens(kind, payload).map((token) => ({ - workspaceId, - itemId, - kind, - token: token.token, - frequency: token.frequency, - })) -} - -function getRawRows(value: RawRowsResult): readonly Row[] { - if (Array.isArray(value)) return value - return (value as { readonly rows: readonly Row[] }).rows -} diff --git a/src/domains/memory/resolve-operations.test.ts b/src/domains/memory/resolve-operations.test.ts deleted file mode 100644 index 9764ccfd..00000000 --- a/src/domains/memory/resolve-operations.test.ts +++ /dev/null @@ -1,378 +0,0 @@ -import { describe, expect, it } from "vitest" - -import type { MemoryOperations } from "./prompts" -import { - resolveMemoryOperations, - toDiffOperation, -} from "./resolve-operations" - -const existingItems = [ - { - id: "item-1", - kind: "indicator_pref", - status: "active", - payload: { - name: "毛利率", - aliases: ["gross margin"], - definition: "毛利占营收的比例", - polarity: "higher_better", - importance: "core", - }, - }, - { id: "item-2", kind: "stance", status: "active" }, - { id: "item-3", kind: "stance", status: "deprecated" }, - { - id: "item-4", - kind: "entity_of_interest", - status: "active", - payload: { - name: "英伟达", - ticker: "NVDA", - aliases: ["NVIDIA"], - knowhereDocumentIds: ["doc-earlier"], - reason: "用户持续跟踪", - }, - }, -] as const - -function makeOperations( - overrides: Partial = {}, -): MemoryOperations { - return { - indicatorPrefs: [], - stances: [], - decisionRules: [], - entities: [], - ...overrides, - } -} - -function makeIndicatorEntry(decision: { - op: "create" | "skip" | "merge" | "deprecate" - targetItemId?: string - reason?: string -}) { - return { - name: "毛利率", - aliases: ["gross margin"], - definition: "毛利占营收的比例", - polarity: "higher_better" as const, - importance: "core" as const, - abstractL0: "用户看重毛利率", - overviewL1: "用户在分析公司时首先看毛利率。", - confidence: 0.9, - decision, - } -} - -describe("resolveMemoryOperations", () => { - it("passes create through and ignores a stray targetItemId", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - makeIndicatorEntry({ op: "create", targetItemId: "item-1" }), - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved).toHaveLength(1) - expect(resolved[0]).toMatchObject({ - op: "create", - kind: "indicator_pref", - payload: { - name: "毛利率", - aliases: ["gross margin"], - polarity: "higher_better", - }, - }) - expect(resolved[0]).not.toHaveProperty("targetItemId") - }) - - it("merges into an existing active item of the same kind", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - makeIndicatorEntry({ op: "merge", targetItemId: "item-1" }), - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "merge", - kind: "indicator_pref", - targetItemId: "item-1", - }) - }) - - it("downgrades merge to skip when the target is missing", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - makeIndicatorEntry({ op: "merge", targetItemId: "item-999" }), - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "skip", - kind: "indicator_pref", - }) - expect(resolved[0]).not.toHaveProperty("targetItemId") - }) - - it("downgrades merge to skip on kind mismatch", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - makeIndicatorEntry({ op: "merge", targetItemId: "item-2" }), - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]?.op).toBe("skip") - }) - - it("downgrades merge to skip when the target is already deprecated", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - stances: [ - { - statement: "长期持有，忽略短期波动", - scope: "投资", - rationale: "用户做长期投资", - abstractL0: "长期投资立场", - overviewL1: "用户强调长期持有。", - confidence: 1, - decision: { op: "merge", targetItemId: "item-3" }, - }, - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]?.op).toBe("skip") - }) - - it("keeps deprecate for an active target", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - stances: [ - { - statement: "美联储短期观点不重要", - scope: "宏观", - rationale: "长期投资", - abstractL0: "不看美联储短期观点", - overviewL1: "用户认为美联储短期观点权重低。", - confidence: 1, - decision: { - op: "deprecate", - targetItemId: "item-2", - reason: "用户改口", - }, - }, - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "deprecate", - targetItemId: "item-2", - reason: "用户改口", - }) - }) - - it("filters entity document ids to the turn's referenced documents", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - entities: [ - { - name: "英伟达", - ticker: "NVDA", - aliases: ["NVIDIA"], - knowhereDocumentIds: ["doc-real", "doc-hallucinated"], - reason: "用户持续跟踪", - abstractL0: "用户关注英伟达", - overviewL1: "用户多次询问英伟达财报。", - confidence: 0.8, - decision: { op: "create" }, - }, - ], - }), - existingItems, - referencedDocumentIds: ["doc-real"], - }) - - expect(resolved[0]).toMatchObject({ - op: "create", - kind: "entity_of_interest", - payload: { - name: "英伟达", - knowhereDocumentIds: ["doc-real"], - }, - }) - }) - - it("unions stored document ids when merging an entity", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - entities: [ - { - name: "英伟达", - ticker: "NVDA", - aliases: ["NVDA Corp"], - knowhereDocumentIds: ["doc-this-turn"], - reason: "用户持续跟踪", - abstractL0: "用户关注英伟达", - overviewL1: "用户多次询问英伟达财报。", - confidence: 0.9, - decision: { op: "merge", targetItemId: "item-4" }, - }, - ], - }), - existingItems, - referencedDocumentIds: ["doc-this-turn"], - }) - - expect(resolved[0]).toMatchObject({ - op: "merge", - targetItemId: "item-4", - payload: { - aliases: ["NVIDIA", "NVDA Corp"], - knowhereDocumentIds: ["doc-earlier", "doc-this-turn"], - }, - }) - }) - - it("unions aliases when merging an indicator preference", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - { - name: "毛利率", - aliases: ["同比毛利率"], - definition: "毛利占营收的比例，也看同比", - polarity: "higher_better" as const, - importance: "core" as const, - abstractL0: "毛利率也看同比", - overviewL1: "用户补充了同比视角。", - confidence: 1, - decision: { op: "merge", targetItemId: "item-1" }, - }, - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "merge", - targetItemId: "item-1", - payload: { - aliases: ["gross margin", "同比毛利率"], - }, - }) - }) - - it("skips create when the payload has no searchable tokens", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - { - name: "!!!", - aliases: [], - definition: "???", - polarity: "context" as const, - importance: "secondary" as const, - abstractL0: "无效符号", - overviewL1: "无法检索。", - confidence: 0.1, - decision: { op: "create" }, - }, - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "skip", - kind: "indicator_pref", - reason: "payload has no searchable tokens", - }) - }) - - it("skips merge when the merged payload has no searchable tokens", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - stances: [ - { - statement: "!!!", - scope: "???", - rationale: "...", - abstractL0: "无效符号", - overviewL1: "无法检索。", - confidence: 0.1, - decision: { op: "merge", targetItemId: "item-2" }, - }, - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(resolved[0]).toMatchObject({ - op: "skip", - kind: "stance", - reason: "merged payload has no searchable tokens", - }) - }) -}) - -describe("toDiffOperation", () => { - it("records create with the inserted item id", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [makeIndicatorEntry({ op: "create" })], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(toDiffOperation(resolved[0]!, "new-id")).toEqual({ - op: "create", - kind: "indicator_pref", - summary: "用户看重毛利率", - itemId: "new-id", - }) - }) - - it("records merge/deprecate with their target item id", () => { - const resolved = resolveMemoryOperations({ - operations: makeOperations({ - indicatorPrefs: [ - makeIndicatorEntry({ op: "merge", targetItemId: "item-1" }), - ], - }), - existingItems, - referencedDocumentIds: [], - }) - - expect(toDiffOperation(resolved[0]!)).toEqual({ - op: "merge", - kind: "indicator_pref", - summary: "用户看重毛利率", - itemId: "item-1", - }) - }) -}) diff --git a/src/domains/memory/resolve-operations.ts b/src/domains/memory/resolve-operations.ts deleted file mode 100644 index ee1e6a42..00000000 --- a/src/domains/memory/resolve-operations.ts +++ /dev/null @@ -1,314 +0,0 @@ -import { buildMemoryItemTokens } from "./search-index" -import type { MemoryOperations } from "./prompts" -import { - parseFluidMemoryPayload, - type FluidMemoryKind, - type FluidMemoryPayload, - type MemoryDiffOperation, -} from "./types" - -/** - * Pure normalization from raw LLM operations to repository-ready - * operations. The LLM output already passed zod validation; this layer - * enforces the invariants the schema cannot express: - * - merge/deprecate must target an existing active item of the same kind - * (otherwise downgraded to skip — conservative, never fabricates) - * - entity knowhereDocumentIds are intersected with the document ids - * actually referenced in the turn (the model cannot invent provenance) - * - create ignores any targetItemId the model may have emitted - * - create/merge payloads must yield at least one lexical token, otherwise - * the item could never be retrieved for later dedup - * - merge unions aliases (and entity document ids) with the target so - * prior search terms / provenance are not wiped by a partial rewrite - */ - -export type ResolvedMemoryOperation = - | { - readonly op: "create" - readonly kind: FluidMemoryKind - readonly payload: FluidMemoryPayload - readonly abstractL0: string - readonly overviewL1: string - readonly confidence: number - readonly summary: string - readonly reason?: string - } - | { - readonly op: "skip" - readonly kind: FluidMemoryKind - readonly summary: string - readonly reason?: string - } - | { - readonly op: "merge" - readonly kind: FluidMemoryKind - readonly targetItemId: string - readonly payload: FluidMemoryPayload - readonly abstractL0: string - readonly overviewL1: string - readonly confidence: number - readonly summary: string - readonly reason?: string - } - | { - readonly op: "deprecate" - readonly kind: FluidMemoryKind - readonly targetItemId: string - readonly summary: string - readonly reason?: string - } - -export type ExistingMemoryItemRef = { - readonly id: string - readonly kind: string - readonly status: string - readonly payload?: unknown -} - -type CandidateEntry = { - readonly abstractL0: string - readonly overviewL1: string - readonly confidence: number - readonly decision: { - readonly op: "create" | "skip" | "merge" | "deprecate" - readonly targetItemId?: string - readonly reason?: string - } -} - -const kindToArrayKey = { - indicator_pref: "indicatorPrefs", - stance: "stances", - decision_rule: "decisionRules", - entity_of_interest: "entities", -} as const - -export function resolveMemoryOperations(input: { - readonly operations: MemoryOperations - readonly existingItems: readonly ExistingMemoryItemRef[] - readonly referencedDocumentIds: readonly string[] -}): ResolvedMemoryOperation[] { - const activeById = new Map( - input.existingItems - .filter((item) => item.status === "active") - .map((item) => [item.id, item] as const), - ) - const allowedDocumentIds = new Set(input.referencedDocumentIds) - - const resolved: ResolvedMemoryOperation[] = [] - - for (const kind of Object.keys(kindToArrayKey) as FluidMemoryKind[]) { - const entries = input.operations[kindToArrayKey[kind]] as readonly (CandidateEntry & - Record)[] - - for (const entry of entries) { - const summary = entry.abstractL0 - const reason = entry.decision.reason - - if (entry.decision.op === "skip") { - resolved.push({ op: "skip", kind, summary, ...(reason ? { reason } : {}) }) - continue - } - - if (entry.decision.op === "merge" || entry.decision.op === "deprecate") { - const targetId = entry.decision.targetItemId - const target = targetId ? activeById.get(targetId) : undefined - if (!target || target.kind !== kind) { - resolved.push({ - op: "skip", - kind, - summary, - reason: `${entry.decision.op} target missing, inactive, or kind mismatch`, - }) - continue - } - if (entry.decision.op === "deprecate") { - resolved.push({ - op: "deprecate", - kind, - targetItemId: target.id, - summary, - ...(reason ? { reason } : {}), - }) - continue - } - const mergePayload = toPayload(kind, entry, allowedDocumentIds) - if (!mergePayload) { - resolved.push({ - op: "skip", - kind, - summary, - reason: "merged payload failed validation", - }) - continue - } - const preserved = preserveFieldsOnMerge( - kind, - mergePayload, - target.payload, - ) - if (!isIndexable(kind, preserved)) { - resolved.push({ - op: "skip", - kind, - summary, - reason: "merged payload has no searchable tokens", - }) - continue - } - resolved.push({ - op: "merge", - kind, - targetItemId: target.id, - payload: preserved, - abstractL0: entry.abstractL0, - overviewL1: entry.overviewL1, - confidence: entry.confidence, - summary, - ...(reason ? { reason } : {}), - }) - continue - } - - const createPayload = toPayload(kind, entry, allowedDocumentIds) - if (!createPayload) { - resolved.push({ - op: "skip", - kind, - summary, - reason: "payload failed validation", - }) - continue - } - if (!isIndexable(kind, createPayload)) { - resolved.push({ - op: "skip", - kind, - summary, - reason: "payload has no searchable tokens", - }) - continue - } - resolved.push({ - op: "create", - kind, - payload: createPayload, - abstractL0: entry.abstractL0, - overviewL1: entry.overviewL1, - confidence: entry.confidence, - summary, - ...(reason ? { reason } : {}), - }) - } - } - - return resolved -} - -function toPayload( - kind: FluidMemoryKind, - entry: Record, - allowedDocumentIds: ReadonlySet, -): FluidMemoryPayload | null { - const candidate: Record = { - name: entry.name, - aliases: entry.aliases, - definition: entry.definition, - polarity: entry.polarity, - importance: entry.importance, - formulaHint: entry.formulaHint, - statement: entry.statement, - scope: entry.scope, - rationale: entry.rationale, - when: entry.when, - then: entry.then, - priority: entry.priority, - ticker: entry.ticker, - reason: entry.reason, - knowhereDocumentIds: Array.isArray(entry.knowhereDocumentIds) - ? entry.knowhereDocumentIds.filter( - (id): id is string => - typeof id === "string" && allowedDocumentIds.has(id), - ) - : [], - } - return parseFluidMemoryPayload(kind, candidate) -} - -/** Reject payloads that could never be found again by the token index. */ -function isIndexable(kind: FluidMemoryKind, payload: FluidMemoryPayload): boolean { - return buildMemoryItemTokens(kind, payload).length > 0 -} - -/** - * Merge replaces the stored payload, but the model only sees this turn. - * Union aliases (and entity document ids) with the target so earlier search - * terms / provenance survive a partial rewrite. - */ -function preserveFieldsOnMerge( - kind: FluidMemoryKind, - mergedPayload: FluidMemoryPayload, - existingPayload: unknown, -): FluidMemoryPayload { - const existing = parseFluidMemoryPayload(kind, existingPayload) - if (!existing) return mergedPayload - - if ( - kind === "indicator_pref" && - "aliases" in mergedPayload && - "aliases" in existing - ) { - return { - ...mergedPayload, - aliases: unionStrings(existing.aliases, mergedPayload.aliases), - } - } - - if ( - kind === "entity_of_interest" && - "aliases" in mergedPayload && - "aliases" in existing && - "knowhereDocumentIds" in mergedPayload && - "knowhereDocumentIds" in existing - ) { - return { - ...mergedPayload, - aliases: unionStrings(existing.aliases, mergedPayload.aliases), - knowhereDocumentIds: unionStrings( - existing.knowhereDocumentIds, - mergedPayload.knowhereDocumentIds, - ), - } - } - - return mergedPayload -} - -function unionStrings( - left: readonly string[], - right: readonly string[], -): string[] { - return [...new Set([...left, ...right])] -} - -/** Diff-audit view of a resolved operation (itemId filled after write). */ -export function toDiffOperation( - operation: ResolvedMemoryOperation, - itemId?: string, -): MemoryDiffOperation { - const base = { - kind: operation.kind, - summary: operation.summary, - ...(operation.reason ? { reason: operation.reason } : {}), - } - switch (operation.op) { - case "create": - return { op: "create", ...base, ...(itemId ? { itemId } : {}) } - case "merge": - return { op: "merge", ...base, itemId: operation.targetItemId } - case "deprecate": - return { op: "deprecate", ...base, itemId: operation.targetItemId } - case "skip": - return { op: "skip", ...base } - } -} diff --git a/src/domains/memory/search-index.test.ts b/src/domains/memory/search-index.test.ts deleted file mode 100644 index cbf5d0b3..00000000 --- a/src/domains/memory/search-index.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { describe, expect, it } from "vitest" - -import { - buildMemoryItemTokens, - buildMemorySearchText, - tokenizeMemoryText, -} from "./search-index" - -describe("tokenizeMemoryText", () => { - it("splits CJK into single characters and latin into words", () => { - expect(tokenizeMemoryText("毛利率 PE gross_margin")).toEqual([ - { token: "毛", frequency: 1 }, - { token: "利", frequency: 1 }, - { token: "率", frequency: 1 }, - { token: "pe", frequency: 1 }, - { token: "gross_margin", frequency: 1 }, - ]) - }) - - it("counts repeated tokens", () => { - expect(tokenizeMemoryText("PE pe Pe")).toEqual([ - { token: "pe", frequency: 3 }, - ]) - }) - - it("returns empty for whitespace-only input", () => { - expect(tokenizeMemoryText(" ")).toEqual([]) - }) -}) - -describe("buildMemorySearchText", () => { - it("indexes indicator name, aliases, and definition", () => { - expect( - buildMemorySearchText("indicator_pref", { - name: "毛利率", - aliases: ["gross margin"], - definition: "毛利除以营收", - polarity: "higher_better", - importance: "core", - }), - ).toBe("毛利率 gross margin 毛利除以营收") - }) - - it("indexes entity name, aliases, and ticker without reason", () => { - expect( - buildMemorySearchText("entity_of_interest", { - name: "英伟达", - aliases: ["NVIDIA"], - ticker: "NVDA", - knowhereDocumentIds: ["doc-1"], - reason: "一直在跟踪", - }), - ).toBe("英伟达 NVIDIA NVDA") - }) - - it("indexes stance statement and scope", () => { - expect( - buildMemorySearchText("stance", { - statement: "做长期投资", - scope: "宏观短期观点", - rationale: "美联储短期说法不重要", - }), - ).toBe("做长期投资 宏观短期观点") - }) - - it("indexes decision rule when and then", () => { - expect( - buildMemorySearchText("decision_rule", { - when: "毛利率连续两季下滑", - then: "减仓观望", - priority: "high", - rationale: "用户明确说过", - }), - ).toBe("毛利率连续两季下滑 减仓观望") - }) -}) - -describe("buildMemoryItemTokens", () => { - it("tokenizes the search text of an item", () => { - const tokens = buildMemoryItemTokens("indicator_pref", { - name: "PE", - aliases: [], - definition: "市盈率", - polarity: "context", - importance: "secondary", - }) - expect(tokens.map((token) => token.token)).toEqual([ - "pe", - "市", - "盈", - "率", - ]) - }) -}) diff --git a/src/domains/memory/search-index.ts b/src/domains/memory/search-index.ts deleted file mode 100644 index 84a53e9b..00000000 --- a/src/domains/memory/search-index.ts +++ /dev/null @@ -1,84 +0,0 @@ -import type { - DecisionRulePayload, - EntityOfInterestPayload, - FluidMemoryKind, - FluidMemoryPayload, - IndicatorPreferencePayload, - StancePayload, -} from "./types" - -/** - * Lexical search-index helpers for fluid memory dedup retrieval. - * - * Pure and dependency-free so both the write path (indexing an item) and the - * read path (turning a turn into a query) share one tokenizer. Tokenization - * mirrors Knowhere map-nav: lowercase, then emit single CJK characters and - * `[a-z0-9_]+` runs. This handles Chinese (no whitespace segmentation) and - * Latin/alphanumeric terms without any Postgres extension. - */ - -export type MemoryToken = { - readonly token: string - readonly frequency: number -} - -// Single CJK char OR a run of latin letters / digits / underscore. -const TOKEN_PATTERN = - /[a-z0-9_]+|[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]/g - -/** - * Build the text that represents an item for lexical matching. Only the - * fields a user would phrase a query against are included (names, aliases, - * short definitions), not provenance or bookkeeping fields. - */ -export function buildMemorySearchText( - kind: FluidMemoryKind, - payload: FluidMemoryPayload, -): string { - return collectSearchParts(kind, payload) - .filter((part) => part.length > 0) - .join(" ") -} - -/** Tokenize free text into deduped tokens with occurrence counts. */ -export function tokenizeMemoryText(text: string): MemoryToken[] { - const counts = new Map() - const matches = text.toLowerCase().match(TOKEN_PATTERN) - if (!matches) return [] - for (const token of matches) { - counts.set(token, (counts.get(token) ?? 0) + 1) - } - return [...counts].map(([token, frequency]) => ({ token, frequency })) -} - -/** Tokens that index one memory item (search text of its payload). */ -export function buildMemoryItemTokens( - kind: FluidMemoryKind, - payload: FluidMemoryPayload, -): MemoryToken[] { - return tokenizeMemoryText(buildMemorySearchText(kind, payload)) -} - -function collectSearchParts( - kind: FluidMemoryKind, - payload: FluidMemoryPayload, -): readonly string[] { - switch (kind) { - case "indicator_pref": { - const p = payload as IndicatorPreferencePayload - return [p.name, ...p.aliases, p.definition] - } - case "stance": { - const p = payload as StancePayload - return [p.statement, p.scope] - } - case "decision_rule": { - const p = payload as DecisionRulePayload - return [p.when, p.then] - } - case "entity_of_interest": { - const p = payload as EntityOfInterestPayload - return [p.name, ...p.aliases, ...(p.ticker ? [p.ticker] : [])] - } - } -} diff --git a/src/domains/memory/service.ts b/src/domains/memory/service.ts deleted file mode 100644 index da772f65..00000000 --- a/src/domains/memory/service.ts +++ /dev/null @@ -1,54 +0,0 @@ -import "server-only" - -import { databaseRuntime } from "@/domains/workspace/database-runtime" -import { memoryRepository } from "./repository" -import type { ResolvedMemoryOperation } from "./resolve-operations" -import type { FluidMemoryKind, MemoryDiffOperation } from "./types" -import type { FluidMemoryItem } from "@/infrastructure/db/schema" - -type MemoryService = { - readonly findDedupCandidates: ( - workspaceId: string, - kind: FluidMemoryKind, - tokens: readonly string[], - limit: number, - ) => Promise - readonly applyOperations: ( - workspaceId: string, - sourceMessageId: string | null, - operations: readonly ResolvedMemoryOperation[], - ) => Promise -} - -const findDedupCandidates: MemoryService["findDedupCandidates"] = ( - workspaceId, - kind, - tokens, - limit, -) => - databaseRuntime.runPromise( - memoryRepository.findDedupCandidatesEffect( - workspaceId, - kind, - tokens, - limit, - ), - ) - -const applyOperations: MemoryService["applyOperations"] = ( - workspaceId, - sourceMessageId, - operations, -) => - databaseRuntime.runPromise( - memoryRepository.applyOperationsEffect( - workspaceId, - sourceMessageId, - operations, - ), - ) - -export const memoryService: MemoryService = { - findDedupCandidates, - applyOperations, -} diff --git a/src/domains/memory/types.ts b/src/domains/memory/types.ts deleted file mode 100644 index dac7602d..00000000 --- a/src/domains/memory/types.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { z } from "zod" - -/** - * Fluid memory type contract. - * - * Four typed payload kinds, extracted from conversation turns. The DB - * stores `payload` as jsonb; these schemas are the validation boundary on - * both write (LLM output) and read (repository decode) paths. - */ - -export const fluidMemoryKinds = [ - "indicator_pref", - "stance", - "decision_rule", - "entity_of_interest", -] as const - -export type FluidMemoryKind = (typeof fluidMemoryKinds)[number] - -export const indicatorPreferencePayloadSchema = z.object({ - name: z.string().min(1), - aliases: z.array(z.string()).default([]), - definition: z.string().min(1), - polarity: z.enum(["higher_better", "lower_better", "context"]), - importance: z.enum(["core", "secondary"]), - formulaHint: z.preprocess( - (value) => (value === null ? undefined : value), - z.string().optional(), - ), -}) - -export const stancePayloadSchema = z.object({ - statement: z.string().min(1), - scope: z.string().min(1), - rationale: z.string().min(1), -}) - -export const decisionRulePayloadSchema = z.object({ - when: z.string().min(1), - then: z.string().min(1), - priority: z.enum(["high", "medium", "low"]), - rationale: z.string().min(1), -}) - -export const entityOfInterestPayloadSchema = z.object({ - name: z.string().min(1), - ticker: z.preprocess( - (value) => (value === null ? undefined : value), - z.string().optional(), - ), - aliases: z.array(z.string()).default([]), - knowhereDocumentIds: z.array(z.string()).default([]), - reason: z.string().min(1), -}) - -export type IndicatorPreferencePayload = z.infer< - typeof indicatorPreferencePayloadSchema -> -export type StancePayload = z.infer -export type DecisionRulePayload = z.infer -export type EntityOfInterestPayload = z.infer< - typeof entityOfInterestPayloadSchema -> - -export type FluidMemoryPayload = - | IndicatorPreferencePayload - | StancePayload - | DecisionRulePayload - | EntityOfInterestPayload - -const payloadSchemas: Record> = { - indicator_pref: indicatorPreferencePayloadSchema, - stance: stancePayloadSchema, - decision_rule: decisionRulePayloadSchema, - entity_of_interest: entityOfInterestPayloadSchema, -} - -export function isFluidMemoryKind(value: unknown): value is FluidMemoryKind { - return ( - typeof value === "string" && - (fluidMemoryKinds as readonly string[]).includes(value) - ) -} - -/** Decode a persisted jsonb payload; returns null when the row is malformed. */ -export function parseFluidMemoryPayload( - kind: FluidMemoryKind, - value: unknown, -): FluidMemoryPayload | null { - const result = payloadSchemas[kind].safeParse(value) - return result.success ? result.data : null -} - -/** One decided operation over the memory set; persisted into memory_diffs. */ -export type MemoryDiffOperation = { - readonly op: "create" | "skip" | "merge" | "deprecate" - readonly kind: FluidMemoryKind - readonly itemId?: string - readonly summary: string - readonly reason?: string -} diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index a3c98054..4e9470f3 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -1,9 +1,7 @@ import { sql } from "drizzle-orm"; import { bigint, - doublePrecision, index, - integer, jsonb, pgTable, text, @@ -340,139 +338,3 @@ export const chatMessages = pgTable( export type ChatMessage = typeof chatMessages.$inferSelect; export type NewChatMessage = typeof chatMessages.$inferInsert; - -/** - * Fluid memory: typed insights extracted from human-AI conversation turns - * (as opposed to "crystal memory", which is the parsed document knowledge - * that stays upstream in Knowhere). - * - * One row per extracted insight. `kind` discriminates the typed `payload` - * (see src/domains/memory/types.ts for the payload contract per kind): - * - indicator_pref — a metric the user cares about (name, aliases, - * polarity, importance) - * - stance — a stated position that shapes judgement - * - decision_rule — a when/then rule over indicators - * - entity_of_interest — a company/topic the user tracks - * - * `abstract_l0` / `overview_l1` are the tiered sidecar summaries (L0 = - * one line for pre-filter/dedup context, L1 = short paragraph for later - * cognition injection). L2 is the payload itself. - * - * Lifecycle: rows start `active`; user revisions deprecate rather than - * delete (conservative merge policy), with `version` bumped on merge. - * - * `source_message_id` points at the assistant message of the turn the - * insight was extracted from; it is set-null on message deletion because - * the insight outlives any single turn. - */ -export const fluidMemoryItems = pgTable( - "fluid_memory_items", - { - id: uuid("id").primaryKey().defaultRandom(), - workspaceId: uuid("workspace_id") - .notNull() - .references(() => workspaces.id, { onDelete: "cascade" }), - kind: text("kind").notNull(), - payload: jsonb("payload").notNull(), - abstractL0: text("abstract_l0").notNull(), - overviewL1: text("overview_l1").notNull(), - sourceMessageId: uuid("source_message_id").references( - () => chatMessages.id, - { onDelete: "set null" }, - ), - confidence: doublePrecision("confidence").notNull(), - status: text("status").notNull(), - version: integer("version").notNull().default(1), - createdAt: timestamp("created_at", { withTimezone: true }) - .notNull() - .defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true }) - .notNull() - .defaultNow(), - }, - (t) => [ - // Workspace lifecycle scans (active vs deprecated). - index("fluid_memory_items_workspace_status_idx").on( - t.workspaceId, - t.status, - ), - index("fluid_memory_items_workspace_kind_idx").on(t.workspaceId, t.kind), - ], -); - -export type FluidMemoryItem = typeof fluidMemoryItems.$inferSelect; -export type NewFluidMemoryItem = typeof fluidMemoryItems.$inferInsert; - -/** - * Lexical inverted index over active fluid memory items, used to retrieve - * dedup candidates at extraction time instead of loading the whole memory - * set into the prompt. One row per (item, token); `frequency` counts token - * occurrences in the item's search text. - * - * Invariant: token rows exist iff the owning item is `active`. Writers keep - * this in sync — create inserts rows, merge replaces them, deprecate deletes - * them — so lookups scan tokens alone (no status join) and never surface a - * deprecated item. - * - * Tokenization mirrors Knowhere map-nav: single CJK characters plus - * `[a-z0-9_]+` runs. Scoring is idf-weighted token overlap computed in SQL, - * keeping the mechanism on portable Postgres (no pg_trgm/pgvector). - */ -export const fluidMemoryTokens = pgTable( - "fluid_memory_tokens", - { - id: uuid("id").primaryKey().defaultRandom(), - workspaceId: uuid("workspace_id") - .notNull() - .references(() => workspaces.id, { onDelete: "cascade" }), - itemId: uuid("item_id") - .notNull() - .references(() => fluidMemoryItems.id, { onDelete: "cascade" }), - kind: text("kind").notNull(), - token: text("token").notNull(), - frequency: integer("frequency").notNull().default(1), - }, - (t) => [ - // Lookup: candidate tokens within a workspace + kind scope. - index("fluid_memory_tokens_lookup_idx").on( - t.workspaceId, - t.kind, - t.token, - ), - // Rebuild/delete a single item's rows on merge/deprecate. - index("fluid_memory_tokens_item_idx").on(t.itemId), - ], -); - -export type FluidMemoryToken = typeof fluidMemoryTokens.$inferSelect; -export type NewFluidMemoryToken = typeof fluidMemoryTokens.$inferInsert; - -/** - * Append-only audit of extraction decisions, one row per processed turn. - * `operations` is a JSONB array of { op, kind, itemId?, summary, reason? } - * records (op = create | skip | merge | deprecate), mirroring OpenViking's - * memory_diff.json so memory growth stays observable and reversible. - */ -export const memoryDiffs = pgTable( - "memory_diffs", - { - id: uuid("id").primaryKey().defaultRandom(), - workspaceId: uuid("workspace_id") - .notNull() - .references(() => workspaces.id, { onDelete: "cascade" }), - sourceMessageId: uuid("source_message_id").references( - () => chatMessages.id, - { onDelete: "set null" }, - ), - operations: jsonb("operations").notNull(), - createdAt: timestamp("created_at", { withTimezone: true }) - .notNull() - .defaultNow(), - }, - (t) => [ - index("memory_diffs_workspace_created_idx").on(t.workspaceId, t.createdAt), - ], -); - -export type MemoryDiff = typeof memoryDiffs.$inferSelect; -export type NewMemoryDiff = typeof memoryDiffs.$inferInsert; diff --git a/src/integrations/knowhere-retrieval-wire.test.ts b/src/integrations/knowhere-retrieval-wire.test.ts new file mode 100644 index 00000000..cf698a2e --- /dev/null +++ b/src/integrations/knowhere-retrieval-wire.test.ts @@ -0,0 +1,90 @@ +import { createServer } from "node:http" +import { once } from "node:events" +import { afterEach, describe, expect, it } from "vitest" +import type { RetrievalQueryParams } from "@ontos-ai/knowhere-sdk" + +import { makeKnowhereClient } from "./knowhere" + +describe("makeKnowhereClient retrieval wire payload", () => { + const originalBaseURL = process.env.KNOWHERE_BASE_URL + + afterEach(() => { + restoreEnv("KNOWHERE_BASE_URL", originalBaseURL) + }) + + it.each<{ + name: string + params: RetrievalQueryParams + body: string + }>([ + { + name: "serializes overlapping include and exclude IDs", + params: { + query: "battery charging", + includeDocumentIds: ["doc_123", "doc_old"], + excludeDocumentIds: ["doc_old"], + } as RetrievalQueryParams, + body: '{"query":"battery charging","include_document_ids":["doc_123","doc_old"],"exclude_document_ids":["doc_old"]}', + }, + { + name: "preserves empty inclusion and exclusion arrays", + params: { + query: "battery charging", + includeDocumentIds: [], + excludeDocumentIds: [], + } as RetrievalQueryParams, + body: '{"query":"battery charging","include_document_ids":[],"exclude_document_ids":[]}', + }, + { + name: "omits document filters when not supplied", + params: { query: "battery charging" }, + body: '{"query":"battery charging"}', + }, + ])("$name", async ({ params, body }) => { + let receivedBody = "" + let receivedMethod: string | undefined + let receivedUrl: string | undefined + const server = createServer((request, response) => { + receivedMethod = request.method + receivedUrl = request.url + request.setEncoding("utf8") + request.on("data", (chunk: string) => { + receivedBody += chunk + }) + request.on("end", () => { + response.writeHead(200, { "Content-Type": "application/json" }) + response.end('{"results":[]}') + }) + }) + + try { + server.listen(0, "127.0.0.1") + await once(server, "listening") + const address = server.address() + if (!address || typeof address === "string") { + throw new Error("Expected TCP server address") + } + process.env.KNOWHERE_BASE_URL = `http://127.0.0.1:${address.port}` + const client = makeKnowhereClient("scope-test") + + await client.retrieval.query(params) + + expect(receivedMethod).toBe("POST") + expect(receivedUrl).toBe("/v2/retrieval/query") + expect(receivedBody).toBe(body) + } finally { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())) + server.closeAllConnections() + }) + } + }) +}) + +function restoreEnv(key: string, value: string | undefined): void { + if (value === undefined) { + delete process.env[key] + return + } + process.env[key] = value +} diff --git a/src/integrations/knowhere.test.ts b/src/integrations/knowhere.test.ts index 74b8c253..a2c4845d 100644 --- a/src/integrations/knowhere.test.ts +++ b/src/integrations/knowhere.test.ts @@ -58,6 +58,7 @@ describe("makeKnowhereClient", () => { expect(constructorSpy).toHaveBeenCalledWith({ apiKey: "sk_test", baseURL: "https://api-staging.knowhereto.ai", + timeout: 120_000, }); }); diff --git a/src/integrations/knowhere.ts b/src/integrations/knowhere.ts index e55fc0bd..a1ca2ed5 100644 --- a/src/integrations/knowhere.ts +++ b/src/integrations/knowhere.ts @@ -37,6 +37,7 @@ export function makeKnowhereClient(apiKey: string): Knowhere { const options: ConstructorParameters[0] = { apiKey, baseURL: process.env.KNOWHERE_BASE_URL, + timeout: 120_000, } const client = new Knowhere(options) return wrapKnowhereClient(client) diff --git a/src/integrations/memento/client.ts b/src/integrations/memento/client.ts new file mode 100644 index 00000000..0b0be90a --- /dev/null +++ b/src/integrations/memento/client.ts @@ -0,0 +1,75 @@ +import "server-only" + +import { + FetchHttpClient, + HttpClient, + HttpClientRequest, +} from "@effect/platform" +import { Effect } from "effect" + +import { summarizeUnknownError } from "@/lib/format-log-value" +import { logger } from "@/lib/logger" +import { readMementoConfig } from "./config" + +export type MementoCaptureInput = { + readonly workspaceId: string + readonly sourceMessageId: string | null + readonly userText: string + readonly assistantText: string + readonly referencedDocumentIds: readonly string[] +} + +export type MementoActivationInput = { + readonly workspaceId: string + readonly unitType: "fluid_memory" | "crystal_chunk" + readonly unitRef: string +} + +export async function captureMemoryTurn( + input: MementoCaptureInput, +): Promise { + try { + await postJson("/turns/capture", input) + } catch (error) { + logger.warn("chat: failed to capture memory turn", { + workspaceId: input.workspaceId, + error: summarizeUnknownError(error), + }) + } +} + +export async function recordActivations( + activations: readonly MementoActivationInput[], +): Promise { + if (activations.length === 0) return + + try { + await postJson("/activations", { activations }) + } catch (error) { + logger.warn("chat: failed to record activations", { + workspaceId: activations[0]?.workspaceId, + count: activations.length, + error: summarizeUnknownError(error), + }) + } +} + +async function postJson(path: string, body: unknown): Promise { + const status = await Effect.runPromise( + Effect.gen(function* () { + const { baseUrl, serviceKey } = readMementoConfig() + const request = yield* HttpClientRequest.post(`${baseUrl}${path}`).pipe( + HttpClientRequest.setHeader( + "Authorization", + `Bearer ${serviceKey}`, + ), + HttpClientRequest.bodyJson(body), + ) + const response = yield* HttpClient.execute(request) + return response.status + }).pipe(Effect.provide(FetchHttpClient.layer)), + ) + if (status < 200 || status >= 300) { + throw new Error(`memento ${path}: HTTP ${status}`) + } +} diff --git a/src/integrations/memento/config.ts b/src/integrations/memento/config.ts new file mode 100644 index 00000000..29eb2343 --- /dev/null +++ b/src/integrations/memento/config.ts @@ -0,0 +1,18 @@ +import "server-only" + +export type MementoConfig = { + readonly baseUrl: string + readonly serviceKey: string +} + +export function readMementoConfig(): MementoConfig { + const baseUrl = process.env.MEMENTO_BASE_URL + const serviceKey = process.env.MEMENTO_SERVICE_KEY + if (!baseUrl) { + throw new Error("MEMENTO_BASE_URL is required.") + } + if (!serviceKey) { + throw new Error("MEMENTO_SERVICE_KEY is required.") + } + return { baseUrl, serviceKey } +} diff --git a/src/integrations/memento/memory-tools.ts b/src/integrations/memento/memory-tools.ts new file mode 100644 index 00000000..052f160f --- /dev/null +++ b/src/integrations/memento/memory-tools.ts @@ -0,0 +1,105 @@ +import "server-only" + +import { Client } from "@modelcontextprotocol/sdk/client/index.js" +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js" + +import type { + MemorySearchItem, + MemorySearchKind, + MemorySearchRequest, + MemorySearchResponse, + MemoryToolRuntime, +} from "@/agent-harness" +import { memorySearchKinds } from "@/agent-harness" +import { readMementoConfig } from "./config" + +type MementoMemoryToolsInput = { + readonly workspaceId: string +} + +export const mementoMemoryTools = { + createRuntime(input: MementoMemoryToolsInput): MemoryToolRuntime { + return { + search: (request) => searchWorkspaceMemory(input.workspaceId, request), + } + }, +} as const + +async function searchWorkspaceMemory( + workspaceId: string, + request: MemorySearchRequest, +): Promise { + const { baseUrl, serviceKey } = readMementoConfig() + const transport = new StreamableHTTPClientTransport(new URL(`${baseUrl}/mcp`), { + requestInit: { + headers: { + Authorization: `Bearer ${serviceKey}`, + }, + }, + }) + const client = new Client({ + name: "knowhere-notebook", + version: "0.1.0", + }) + await client.connect(transport) + try { + const result = await client.callTool({ + name: "memory_search", + arguments: { + workspaceId, + query: request.query, + ...(request.kinds ? { kinds: request.kinds } : {}), + }, + }) + if (result.isError) { + throw new Error("memento memory_search failed") + } + return parseMemorySearchResponse(result.structuredContent) + } finally { + await client.close() + } +} + +function parseMemorySearchResponse(value: unknown): MemorySearchResponse { + if (!value || typeof value !== "object") { + throw new Error("memento memory_search returned no structured content") + } + const record = value as Record + if (typeof record.query !== "string" || !Array.isArray(record.items)) { + throw new Error("memento memory_search structured content is invalid") + } + return { + query: record.query, + items: record.items.map(parseMemorySearchItem), + } +} + +function parseMemorySearchItem(value: unknown): MemorySearchItem { + if (!value || typeof value !== "object") { + throw new Error("memento memory_search item is invalid") + } + const item = value as Record + if ( + typeof item.ref !== "string" || + typeof item.itemId !== "string" || + typeof item.abstractL0 !== "string" || + typeof item.overviewL1 !== "string" || + !isMemorySearchKind(item.kind) + ) { + throw new Error("memento memory_search item is invalid") + } + return { + ref: item.ref, + itemId: item.itemId, + kind: item.kind, + abstractL0: item.abstractL0, + overviewL1: item.overviewL1, + } +} + +function isMemorySearchKind(value: unknown): value is MemorySearchKind { + return ( + typeof value === "string" && + (memorySearchKinds as readonly string[]).includes(value) + ) +}