From 9c8ab785045b557fe0e2b369f7dcfda4bf9bde40 Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Fri, 11 Sep 2026 17:18:47 +0800 Subject: [PATCH 1/2] feat(sources): add chunkCount to source handling and update related tests - Introduced chunkCount property in various source-related files to enhance source management. - Updated the upsertMaterializedDemoSourceEffect to handle chunkCount during source insertion. - Modified tests across multiple files to include chunkCount, ensuring consistency and coverage. - Removed unused functions and types related to chunk counting to streamline the codebase. This enhancement improves the handling of source chunk counts, facilitating better data management and retrieval. --- drizzle/0018_source_chunk_count.sql | 1 + drizzle/meta/0018_snapshot.json | 938 ++++++++++++++++++ drizzle/meta/_journal.json | 7 + notebook-deferred-issues-2026-09-11.md | 36 + .../demo-sources/materialize/route.test.ts | 4 +- src/app/api/demo-sources/materialize/route.ts | 3 +- .../sources/[sourceId]/chunks/route.test.ts | 2 + src/app/api/sources/[sourceId]/route.test.ts | 2 + src/app/api/sources/route.test.ts | 1 + src/domains/chat/index.test.ts | 1 + .../chat/media-asset-hardening.test.ts | 1 + src/domains/chat/media-assets.test.ts | 1 + src/domains/chat/page-citation-assets.test.ts | 1 + src/domains/chat/route-service.test.ts | 1 + src/domains/chat/service.test.ts | 1 + src/domains/chunks/index.test.ts | 1 + .../demo/workspace-source-resolution.ts | 30 - src/domains/sources/counts.test.ts | 335 ------- src/domains/sources/counts.ts | 203 ---- src/domains/sources/demo-source-repository.ts | 7 + src/domains/sources/reconcile.test.ts | 1 + src/domains/sources/remote-library.test.ts | 1 + src/domains/sources/repository.ts | 2 + src/domains/sources/retry.test.ts | 1 + src/domains/sources/route-dependencies.ts | 7 - src/domains/sources/route-listing.ts | 26 +- src/domains/sources/route-retry.test.ts | 1 + src/domains/sources/route-service.test.ts | 51 +- src/domains/sources/route-types.ts | 9 - .../source-reconcile-route-workflow.test.ts | 42 + .../source-reconcile-route-workflow.ts | 36 +- .../sources/source-reconcile-workflow.test.ts | 1 + .../sources/source-row-repository.test.ts | 1 + src/domains/sources/source-row-repository.ts | 17 + src/domains/sources/upload.test.ts | 1 + src/domains/sources/view.test.ts | 5 +- src/domains/sources/view.ts | 9 +- src/domains/sources/workflow-runtime.test.ts | 1 + src/domains/sources/workflow-runtime.ts | 19 + src/domains/workspace/initial-state.test.ts | 69 +- src/domains/workspace/initial-state.ts | 42 +- src/infrastructure/db/schema.ts | 5 + 42 files changed, 1151 insertions(+), 772 deletions(-) create mode 100644 drizzle/0018_source_chunk_count.sql create mode 100644 drizzle/meta/0018_snapshot.json create mode 100644 notebook-deferred-issues-2026-09-11.md delete mode 100644 src/domains/sources/counts.test.ts delete mode 100644 src/domains/sources/counts.ts diff --git a/drizzle/0018_source_chunk_count.sql b/drizzle/0018_source_chunk_count.sql new file mode 100644 index 00000000..dedf1511 --- /dev/null +++ b/drizzle/0018_source_chunk_count.sql @@ -0,0 +1 @@ +ALTER TABLE "sources" ADD COLUMN "chunk_count" integer; \ No newline at end of file diff --git a/drizzle/meta/0018_snapshot.json b/drizzle/meta/0018_snapshot.json new file mode 100644 index 00000000..fdec2c5a --- /dev/null +++ b/drizzle/meta/0018_snapshot.json @@ -0,0 +1,938 @@ +{ + "id": "01b8a746-ca75-4e79-adab-098a03a39e0e", + "prevId": "d4de5ce8-e16b-4dc2-a9ad-2b76bbca8150", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.chat_messages": { + "name": "chat_messages", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "thread_id": { + "name": "thread_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "citations": { + "name": "citations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "artifacts": { + "name": "artifacts", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "chat_messages_thread_created_idx": { + "name": "chat_messages_thread_created_idx", + "columns": [ + { + "expression": "thread_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_messages_thread_id_chat_threads_id_fk": { + "name": "chat_messages_thread_id_chat_threads_id_fk", + "tableFrom": "chat_messages", + "tableTo": "chat_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.chat_threads": { + "name": "chat_threads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "chat_threads_workspace_updated_idx": { + "name": "chat_threads_workspace_updated_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "chat_threads_workspace_demo_key_idx": { + "name": "chat_threads_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "chat_threads_workspace_id_workspaces_id_fk": { + "name": "chat_threads_workspace_id_workspaces_id_fk", + "tableFrom": "chat_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.demo_source_visibilities": { + "name": "demo_source_visibilities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "demo_source_id": { + "name": "demo_source_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hidden_at": { + "name": "hidden_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "demo_source_visibilities_workspace_source_idx": { + "name": "demo_source_visibilities_workspace_source_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "demo_source_visibilities_workspace_idx": { + "name": "demo_source_visibilities_workspace_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "demo_source_visibilities_workspace_id_workspaces_id_fk": { + "name": "demo_source_visibilities_workspace_id_workspaces_id_fk", + "tableFrom": "demo_source_visibilities", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.parsed_document_sync_leases": { + "name": "parsed_document_sync_leases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "document_id": { + "name": "document_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_token": { + "name": "lease_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "release_reason": { + "name": "release_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "parsed_document_sync_leases_token_idx": { + "name": "parsed_document_sync_leases_token_idx", + "columns": [ + { + "expression": "lease_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_active_idx": { + "name": "parsed_document_sync_leases_active_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_workspace_active_idx": { + "name": "parsed_document_sync_leases_workspace_active_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "parsed_document_sync_leases_document_active_idx": { + "name": "parsed_document_sync_leases_document_active_idx", + "columns": [ + { + "expression": "document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "released_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "parsed_document_sync_leases_workspace_id_workspaces_id_fk": { + "name": "parsed_document_sync_leases_workspace_id_workspaces_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parsed_document_sync_leases_source_id_sources_id_fk": { + "name": "parsed_document_sync_leases_source_id_sources_id_fk", + "tableFrom": "parsed_document_sync_leases", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.source_parse_results": { + "name": "source_parse_results", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "result_blob_url": { + "name": "result_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_url": { + "name": "snapshot_manifest_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "snapshot_manifest_key": { + "name": "snapshot_manifest_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revision_key": { + "name": "revision_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "asset_urls": { + "name": "asset_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "source_parse_results_source_id_idx": { + "name": "source_parse_results_source_id_idx", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "source_parse_results_source_id_sources_id_fk": { + "name": "source_parse_results_source_id_sources_id_fk", + "tableFrom": "source_parse_results", + "tableTo": "sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_parse_results_source_id_unique": { + "name": "source_parse_results_source_id_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sources": { + "name": "sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "workspace_id": { + "name": "workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "failure_stage": { + "name": "failure_stage", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_job_id": { + "name": "knowhere_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "knowhere_document_id": { + "name": "knowhere_document_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_pathname": { + "name": "staged_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "staged_blob_url": { + "name": "staged_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_pathname": { + "name": "original_blob_pathname", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "original_blob_url": { + "name": "original_blob_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_key": { + "name": "demo_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "chunk_count": { + "name": "chunk_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "sources_workspace_created_idx": { + "name": "sources_workspace_created_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_status_idx": { + "name": "sources_workspace_status_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_demo_key_idx": { + "name": "sources_workspace_demo_key_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "demo_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "sources_workspace_document_idx": { + "name": "sources_workspace_document_idx", + "columns": [ + { + "expression": "workspace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "knowhere_document_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "knowhere_document_id IS NOT NULL AND deleted_at IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sources_workspace_id_workspaces_id_fk": { + "name": "sources_workspace_id_workspaces_id_fk", + "tableFrom": "sources", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workspaces": { + "name": "workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "workspaces_user_id_idx": { + "name": "workspaces_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "workspaces_user_id_unique": { + "name": "workspaces_user_id_unique", + "nullsNotDistinct": false, + "columns": [ + "user_id" + ] + }, + "workspaces_namespace_unique": { + "name": "workspaces_namespace_unique", + "nullsNotDistinct": false, + "columns": [ + "namespace" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 4ee28af6..84390235 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -127,6 +127,13 @@ "when": 1789017481605, "tag": "0017_overjoyed_shooting_star", "breakpoints": true + }, + { + "idx": 18, + "version": "7", + "when": 1789118032723, + "tag": "0018_source_chunk_count", + "breakpoints": true } ] } \ No newline at end of file diff --git a/notebook-deferred-issues-2026-09-11.md b/notebook-deferred-issues-2026-09-11.md new file mode 100644 index 00000000..ae8440a0 --- /dev/null +++ b/notebook-deferred-issues-2026-09-11.md @@ -0,0 +1,36 @@ +# Notebook 文档范围对齐记录（2026-09-11） + +## 原始问题 + +- 原始状态：代码链路已确认；未复现用户实际请求受影响。用户随后批准接通包含、排除两种范围，覆盖问题 2 的文档范围部分和问题 3。 +- Notebook 本地版本：`99cfe1d`；Knowhere 本地版本：`bf94dbe5`。 +- 实际路径：来源勾选状态 → `excludedSourceIds` → `excludeDocuments` 转为 `excludeDocumentIds` → Knowhere API 的 `exclude_document_ids` → retrieval context。 +- Knowhere 已接收参数。进入 agent_explore 时，`run_episode` 和 `ToolContext` 未传递该排除集合；最终 `assemble_retrieval_results` 才过滤结果。 +- 示例：排除 A 后，agent 仍可能读 A，最终 A 的证据被删除；是否实际发生、是否影响答案尚未验证。 +- 批准方案：将请求级包含、排除集合传到 Explore 工具上下文，各工具数据库查询在返回给模型前执行过滤，保留最终过滤；小语料和朴素检索执行同样范围。不增加模型调用，agent 在允许范围内继续自由探索。 +- 验证：排除 A 时，list/recall/grep/node_filter/read/assets/neighbors 均不能返回 A；未排除 B 仍能正常探索。空排除集合保持原行为。 +- 本次实现覆盖显式文档包含和排除；自然语言筛选仍由 agent 选择 node_filter 等工具执行。 + +代码位置： + +- Notebook：`src/components/workspace-chat-workflow.ts`、`src/domains/chat/retrieval.ts`、`src/domains/chat/index.ts`。 +- Knowhere：`apps/api/app/api/v1/routes/retrieval.py`、`packages/shared-python/shared/services/retrieval/execution/routes.py`、`packages/shared-python/shared/services/retrieval/agent_explore/dispatch.py`、`packages/shared-python/shared/services/retrieval/agent_tools/registry.py`。 + +## 参数约定与实现 + +- `includeDocumentIds` / `include_document_ids`：不传表示不限制，`[]` 表示空范围；只检索列表中的文档。 +- `excludeDocumentIds` / `exclude_document_ids`：排除列表中的文档；排除优先。 +- Notebook：现有 `knowhere_search` 工具增加两个字段；与资料排除状态合并。仅接受已有资料或本轮先前检索结果中确认的文档 ID，未知名称仍随自然语言 query 交给 Knowhere 定位。 +- SDK：新增包含参数类型与文档，HTTP 序列化保留空数组。Notebook 当前安装的 SDK 已用真实本机 HTTP 请求验证能正确发送，无需写入本地依赖路径。 +- Knowhere：统一 `DocumentScope` 作用于三种检索路径、八个 corpus 工具、最终引用和关联资源；缓存区分未限制、空集合、指定集合。 +- 清理仅限本次范围：Notebook 合并重复搜索请求类型；Knowhere 移除 recall 为限定文档而先查全库补集的旧转换。 +- 不涉及 DeepSeek 收工门禁、模型配置、路径/阈值语义或前端展示机制。 + +## 验证记录 + +- Notebook：聊天、账本、引用和媒体相关 193 项测试通过；TypeScript、ESLint、diff 检查通过。 +- SDK：61 项测试通过，包含两种认证方式下的实际 HTTP 字段检查；类型、lint 和格式检查通过。 +- Knowhere：42 项真实 PostgreSQL 合约测试、27 项共享测试通过；最后兼容性调整后，15 项范围合约测试再次通过。Pyright、Ruff、diff 检查通过。 +- 两种 Explore harness 使用真实执行循环和数据库工具，模型 provider 使用模拟回复；未调用线上模型，未验证其自然语言选择行为。数据库测试使用隔离测试库。 +- 尚未提交、发布或部署；线上需后端发布后才会执行新增范围语义。 +- Effect 指南：已查阅 `basics`、`testing`；保留现有 Effect 执行方式。 diff --git a/src/app/api/demo-sources/materialize/route.test.ts b/src/app/api/demo-sources/materialize/route.test.ts index 0818dca7..3a3be24d 100644 --- a/src/app/api/demo-sources/materialize/route.test.ts +++ b/src/app/api/demo-sources/materialize/route.test.ts @@ -60,7 +60,7 @@ describe("POST /api/demo-sources/materialize", () => { }, ]) mocks.upsertMaterializedDemoSource.mockResolvedValue( - makeSource(workspace.id), + makeSource(workspace.id, { chunkCount: 70 }), ) const response = await POST( @@ -109,6 +109,7 @@ describe("POST /api/demo-sources/materialize", () => { sizeBytes: 5648867, knowhereDocumentId: "doc_user_copy", originalBlobUrl: "https://example.com/tsla-q4-2025.pdf", + chunkCount: 70, }, ) }) @@ -240,6 +241,7 @@ function makeSource( originalBlobPathname: null, originalBlobUrl: "https://example.com/tsla-q4-2025.pdf", demoKey: "demo-tsla-q4-2025", + chunkCount: null, createdAt: new Date("2026-05-10T00:00:00.000Z"), updatedAt: new Date("2026-05-10T00:00:00.000Z"), deletedAt: null, diff --git a/src/app/api/demo-sources/materialize/route.ts b/src/app/api/demo-sources/materialize/route.ts index c87a1dbb..ae792ae9 100644 --- a/src/app/api/demo-sources/materialize/route.ts +++ b/src/app/api/demo-sources/materialize/route.ts @@ -71,9 +71,10 @@ export async function POST(request: Request): Promise { sizeBytes: source.sizeBytes, knowhereDocumentId: source.documentId, originalBlobUrl: demoOriginalFile.getPublicUrl(source), + chunkCount: source.chunkCount, }), ) - return toSourceView(row, { chunkCount: source.chunkCount }) + return toSourceView(row) }), ), { concurrency: "unbounded" }, diff --git a/src/app/api/sources/[sourceId]/chunks/route.test.ts b/src/app/api/sources/[sourceId]/chunks/route.test.ts index f4102859..e38d9a8f 100644 --- a/src/app/api/sources/[sourceId]/chunks/route.test.ts +++ b/src/app/api/sources/[sourceId]/chunks/route.test.ts @@ -311,6 +311,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { originalBlobPathname: null, originalBlobUrl: "/api/demo-sources/demo-tsla-q4-2025/original", demoKey: "demo-tsla-q4-2025", + chunkCount: null, createdAt: new Date("2026-05-10T00:00:00.000Z"), updatedAt: new Date("2026-05-10T00:00:00.000Z"), deletedAt: null, @@ -845,6 +846,7 @@ function makeReadySource(overrides: Record) { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-10T00:00:00.000Z"), updatedAt: new Date("2026-05-10T00:00:00.000Z"), deletedAt: null, diff --git a/src/app/api/sources/[sourceId]/route.test.ts b/src/app/api/sources/[sourceId]/route.test.ts index f531c232..b9cbcd60 100644 --- a/src/app/api/sources/[sourceId]/route.test.ts +++ b/src/app/api/sources/[sourceId]/route.test.ts @@ -343,6 +343,7 @@ describe("PATCH /api/sources/[sourceId]", () => { originalBlobUrl: "https://store.public.blob.vercel-storage.com/source-uploads/upload_1/document.pdf", demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-10T00:00:00Z"), updatedAt: new Date("2026-05-10T00:00:00Z"), deletedAt: null, @@ -375,6 +376,7 @@ describe("PATCH /api/sources/[sourceId]", () => { originalBlobUrl: "https://store.public.blob.vercel-storage.com/source-uploads/upload_1/document.pdf", demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-10T00:00:00Z"), updatedAt: new Date("2026-05-10T00:00:00Z"), deletedAt: null, diff --git a/src/app/api/sources/route.test.ts b/src/app/api/sources/route.test.ts index a7b10c6c..154efa20 100644 --- a/src/app/api/sources/route.test.ts +++ b/src/app/api/sources/route.test.ts @@ -75,6 +75,7 @@ const source: Source = { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-10T00:00:00Z"), updatedAt: new Date("2026-05-10T00:00:00Z"), deletedAt: null, diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index 26fd0385..48203ff4 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -3284,6 +3284,7 @@ function makeSource(overrides: Partial = {}): Source { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-06T00:00:00Z"), updatedAt: new Date("2026-05-06T00:00:00Z"), deletedAt: null, diff --git a/src/domains/chat/media-asset-hardening.test.ts b/src/domains/chat/media-asset-hardening.test.ts index 2afa6d93..2660cfa6 100644 --- a/src/domains/chat/media-asset-hardening.test.ts +++ b/src/domains/chat/media-asset-hardening.test.ts @@ -208,6 +208,7 @@ function makeSource(overrides: Partial = {}): Source { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-06T00:00:00Z"), updatedAt: new Date("2026-05-06T00:00:00Z"), deletedAt: null, diff --git a/src/domains/chat/media-assets.test.ts b/src/domains/chat/media-assets.test.ts index c4ee72f5..7f15318c 100644 --- a/src/domains/chat/media-assets.test.ts +++ b/src/domains/chat/media-assets.test.ts @@ -319,6 +319,7 @@ function makeSource(overrides: Partial = {}): Source { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-06-04T00:00:00Z"), updatedAt: new Date("2026-06-04T00:00:00Z"), deletedAt: null, diff --git a/src/domains/chat/page-citation-assets.test.ts b/src/domains/chat/page-citation-assets.test.ts index 06f679ed..65be327c 100644 --- a/src/domains/chat/page-citation-assets.test.ts +++ b/src/domains/chat/page-citation-assets.test.ts @@ -265,6 +265,7 @@ function makeSource(overrides: Partial = {}): Source { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-07-03T00:00:00.000Z"), updatedAt: new Date("2026-07-03T00:00:00.000Z"), deletedAt: null, diff --git a/src/domains/chat/route-service.test.ts b/src/domains/chat/route-service.test.ts index 80d4f7bd..1db1e94a 100644 --- a/src/domains/chat/route-service.test.ts +++ b/src/domains/chat/route-service.test.ts @@ -1084,6 +1084,7 @@ function makeSource(overrides: Partial = {}): Source { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-06T00:00:00Z"), updatedAt: new Date("2026-05-06T00:00:00Z"), deletedAt: null, diff --git a/src/domains/chat/service.test.ts b/src/domains/chat/service.test.ts index 4aa1a5a0..620fa9f7 100644 --- a/src/domains/chat/service.test.ts +++ b/src/domains/chat/service.test.ts @@ -305,6 +305,7 @@ function makeSource(overrides: Partial = {}): Source { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-06T00:00:00Z"), updatedAt: new Date("2026-05-06T00:00:00Z"), deletedAt: null, diff --git a/src/domains/chunks/index.test.ts b/src/domains/chunks/index.test.ts index 873bd130..91f3021f 100644 --- a/src/domains/chunks/index.test.ts +++ b/src/domains/chunks/index.test.ts @@ -704,6 +704,7 @@ function makeSource(overrides: Partial = {}): Source { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-06T00:00:00Z"), updatedAt: new Date("2026-05-06T00:00:00Z"), deletedAt: null, diff --git a/src/domains/demo/workspace-source-resolution.ts b/src/domains/demo/workspace-source-resolution.ts index d0918afa..e14339f8 100644 --- a/src/domains/demo/workspace-source-resolution.ts +++ b/src/domains/demo/workspace-source-resolution.ts @@ -6,10 +6,6 @@ type WorkspaceDemoSourceResolution = { readonly workspaceSources: readonly Source[] } -type SourceViewOptions = { - readonly chunkCount?: number -} - export function resolveWorkspaceDemoSources( sources: readonly Source[], catalog: DemoCatalog, @@ -39,32 +35,6 @@ export function resolveWorkspaceDemoSources( } } -export function getWorkspaceSourcesNeedingChunkCount( - sources: readonly Source[], -): Source[] { - return sources.filter((source) => !source.demoKey) -} - -export function getMaterializedDemoSourceViewOptionsBySourceId( - sources: readonly Source[], - catalog: DemoCatalog, -): ReadonlyMap { - const chunkCountByDemoSourceId: ReadonlyMap = new Map( - catalog.sources.map((source) => [source.demoSourceId, source.chunkCount]), - ) - - return new Map( - sources.flatMap((source): readonly [string, SourceViewOptions][] => { - if (!source.demoKey) return [] - - const chunkCount = chunkCountByDemoSourceId.get(source.demoKey) - if (chunkCount === undefined) return [] - - return [[source.id, { chunkCount }]] - }), - ) -} - function isLegacyCanonicalDemoSource( source: Source, canonicalDocumentIdByDemoSourceId: ReadonlyMap, diff --git a/src/domains/sources/counts.test.ts b/src/domains/sources/counts.test.ts deleted file mode 100644 index 72d132f9..00000000 --- a/src/domains/sources/counts.test.ts +++ /dev/null @@ -1,335 +0,0 @@ -import { describe, expect, it, vi } from "vitest" -import { Effect } from "effect" - -import type Knowhere from "@ontos-ai/knowhere-sdk" -import type { Knowledge } from "@ontos-ai/knowhere-sdk" - -import type { Source } from "@/infrastructure/db/schema" - -function makeSource(overrides: Partial = {}): Source { - return { - id: "source_1", - workspaceId: "workspace_1", - title: "notes.pdf", - mimeType: "application/pdf", - sizeBytes: 1, - status: "ready", - failureReason: null, - failureStage: null, - knowhereJobId: "job_1", - knowhereDocumentId: "doc_1", - stagedBlobPathname: null, - stagedBlobUrl: null, - originalBlobPathname: null, - originalBlobUrl: null, - demoKey: null, - createdAt: new Date("2026-05-06T00:00:00Z"), - updatedAt: new Date("2026-05-06T00:00:00Z"), - deletedAt: null, - ...overrides, - } -} - -describe("countChunksBySourceId", () => { - it("counts ready source chunks from the document total", async () => { - const listChunks = vi.fn(async () => ({ pagination: { total: 12 } })) - const readChunks = vi.fn(async () => ({ - chunks: [], - totalChunks: 0, - })) - const mockClient = { - documents: { listChunks }, - knowledge: { readChunks }, - } as unknown as Knowhere - - const { countChunksBySourceId } = await import("./counts") - - const counts = await Effect.runPromise( - countChunksBySourceId( - [ - makeSource({ id: "ready", knowhereDocumentId: "doc_ready" }), - makeSource({ - id: "parsing", - status: "parsing", - knowhereDocumentId: null, - }), - makeSource({ id: "missing-doc", knowhereDocumentId: null }), - ], - mockClient, - ), - ) - - expect(listChunks).toHaveBeenCalledTimes(1) - expect(readChunks).toHaveBeenCalledWith({ - documentId: "doc_ready", - revisionKey: "job_1", - chunkType: "page", - page: 1, - pageSize: 1, - assetUrlPolicy: "durable", - }) - expect(listChunks).toHaveBeenCalledWith("doc_ready", { - page: 1, - pageSize: 1, - }) - expect(counts).toEqual(new Map([["ready", 12]])) - }) - - it("skips a source count when the document total lookup fails", async () => { - const listChunks = vi.fn(async () => { - throw new Error("temporary outage") - }) - const readChunks = vi.fn(async () => ({ chunks: [], totalChunks: 0 })) - const mockClient = { - documents: { listChunks }, - knowledge: { readChunks }, - } as unknown as Knowhere - - const { countChunksBySourceId } = await import("./counts") - - const counts = await Effect.runPromise( - countChunksBySourceId( - [makeSource({ id: "ready", knowhereDocumentId: "doc_ready" })], - mockClient, - ), - ) - - expect(counts.size).toBe(0) - expect(listChunks).toHaveBeenCalledTimes(1) - }) - - it("does not count materialized demo sources through their copied document id", async () => { - const listChunks = vi.fn().mockResolvedValue({ - pagination: { total: 70 }, - }) - const readChunks = vi.fn(async () => ({ chunks: [], totalChunks: 0 })) - const mockClient = { - documents: { listChunks }, - knowledge: { readChunks }, - } as unknown as Knowhere - - const { countChunksBySourceId } = await import("./counts") - - const counts = await Effect.runPromise( - countChunksBySourceId( - [ - makeSource({ - id: "source_demo", - demoKey: "demo-tsla-q4-2025", - knowhereDocumentId: "doc_user_copy", - }), - ], - mockClient, - ), - ) - - expect(listChunks).not.toHaveBeenCalled() - expect(readChunks).not.toHaveBeenCalled() - expect(counts.size).toBe(0) - }) -}) - -describe("sourceViewOptionsBySourceId", () => { - it("detects page count from many page assets in a single SDK page chunk", async () => { - const listChunks = vi.fn(async () => ({ pagination: { total: 12 } })) - const readChunks = vi.fn(async () => ({ - chunks: [ - { - chunkId: "page_bundle", - chunkType: "page", - metadata: { - pageAssets: Array.from({ length: 20 }, (_, index) => ({ - pageNum: index + 1, - artifactRef: `pages/page-${String(index + 1).padStart(6, "0")}.png`, - assetUrl: `https://assets.example/page-${index + 1}.png`, - })), - }, - }, - ], - totalChunks: 1, - })) - const mockClient = { - documents: { listChunks }, - knowledge: { readChunks }, - } as unknown as Knowhere - - const { sourceViewOptionsBySourceId } = await import("./counts") - - const options = await Effect.runPromise( - sourceViewOptionsBySourceId( - [makeSource({ id: "ready", knowhereDocumentId: "doc_ready" })], - mockClient, - ), - ) - - expect(options.get("ready")).toEqual({ - chunkCount: 20, - documentPresentation: { kind: "page-assets", pageCount: 20 }, - }) - expect(listChunks).not.toHaveBeenCalled() - }) - - it("detects page-asset documents from SDK page chunks", async () => { - const listChunks = vi.fn(async () => ({ pagination: { total: 12 } })) - const readChunks = vi.fn(async () => ({ - chunks: [ - { - chunkId: "page_1", - chunkType: "page", - metadata: { - pageAssets: [ - { - pageNum: 1, - artifactRef: "pages/page-000001.png", - assetUrl: "https://assets.example/page-000001.png", - }, - ], - }, - }, - ], - totalChunks: 4, - })) - const mockClient = { - documents: { listChunks }, - knowledge: { readChunks }, - } as unknown as Knowhere - - const { sourceViewOptionsBySourceId } = await import("./counts") - - const options = await Effect.runPromise( - sourceViewOptionsBySourceId( - [makeSource({ id: "ready", knowhereDocumentId: "doc_ready" })], - mockClient, - ), - ) - - expect(options.get("ready")).toEqual({ - chunkCount: 4, - documentPresentation: { kind: "page-assets", pageCount: 4 }, - }) - expect(listChunks).not.toHaveBeenCalled() - }) - - it("uses a source-specific knowledge reader for presentation detection", async () => { - const defaultReadChunks = vi.fn(async () => ({ chunks: [], totalChunks: 0 })) - const parsedStorageReadChunks = vi.fn(async () => ({ - chunks: [ - { - chunkId: "page_1", - chunkType: "page", - metadata: { - pageAssets: [ - { - pageNum: 1, - artifactRef: "pages/page-000001.png", - assetUrl: "https://assets.example/page-000001.png", - }, - ], - }, - }, - ], - totalChunks: 1, - })) - const listChunks = vi.fn(async () => ({ pagination: { total: 12 } })) - const mockClient = { - documents: { listChunks }, - knowledge: { readChunks: defaultReadChunks }, - } as unknown as Knowhere - - const { sourceViewOptionsBySourceId } = await import("./counts") - - const options = await Effect.runPromise( - sourceViewOptionsBySourceId( - [makeSource({ id: "ready", knowhereDocumentId: "doc_ready" })], - mockClient, - { - getKnowledgeForSource: () => - ({ readChunks: parsedStorageReadChunks }) as unknown as Knowledge, - }, - ), - ) - - expect(options.get("ready")).toEqual({ - chunkCount: 1, - documentPresentation: { kind: "page-assets", pageCount: 1 }, - }) - expect(parsedStorageReadChunks).toHaveBeenCalledWith({ - documentId: "doc_ready", - revisionKey: "job_1", - chunkType: "page", - page: 1, - pageSize: 1, - assetUrlPolicy: "durable", - }) - expect(defaultReadChunks).not.toHaveBeenCalled() - }) - - it("falls back to chunk counts when page presentation detection fails", async () => { - const listChunks = vi.fn(async () => ({ pagination: { total: 12 } })) - const readChunks = vi.fn(async () => { - throw new Error("parsed storage and remote unavailable") - }) - const mockClient = { - documents: { listChunks }, - knowledge: { readChunks }, - } as unknown as Knowhere - - const { sourceViewOptionsBySourceId } = await import("./counts") - - const options = await Effect.runPromise( - sourceViewOptionsBySourceId( - [makeSource({ id: "ready", knowhereDocumentId: "doc_ready" })], - mockClient, - ), - ) - - expect(options.get("ready")).toEqual({ chunkCount: 12 }) - expect(listChunks).toHaveBeenCalledWith("doc_ready", { - page: 1, - pageSize: 1, - }) - }) - - it("skips page presentation reads when detection is disabled", async () => { - const listChunks = vi.fn(async () => ({ pagination: { total: 12 } })) - const readChunks = vi.fn(async () => ({ - chunks: [ - { - chunkId: "page_1", - chunkType: "page", - metadata: { - pageAssets: [ - { - pageNum: 1, - artifactRef: "pages/page-000001.png", - assetUrl: "https://assets.example/page-000001.png", - }, - ], - }, - }, - ], - totalChunks: 1, - })) - const mockClient = { - documents: { listChunks }, - knowledge: { readChunks }, - } as unknown as Knowhere - - const { sourceViewOptionsBySourceId } = await import("./counts") - - const options = await Effect.runPromise( - sourceViewOptionsBySourceId( - [makeSource({ id: "ready", knowhereDocumentId: "doc_ready" })], - mockClient, - { documentPresentationDetection: "disabled" }, - ), - ) - - expect(readChunks).not.toHaveBeenCalled() - expect(listChunks).toHaveBeenCalledWith("doc_ready", { - page: 1, - pageSize: 1, - }) - expect(options.get("ready")).toEqual({ chunkCount: 12 }) - }) -}) diff --git a/src/domains/sources/counts.ts b/src/domains/sources/counts.ts deleted file mode 100644 index d161adf6..00000000 --- a/src/domains/sources/counts.ts +++ /dev/null @@ -1,203 +0,0 @@ -import "server-only" - -import { Effect } from "effect" -import type Knowhere from "@ontos-ai/knowhere-sdk" -import type { Knowledge, KnowledgeReadChunk } from "@ontos-ai/knowhere-sdk" - -import type { Source } from "@/infrastructure/db/schema" -import type { SourceDocumentPresentation } from "./types" - -type PageAssetDocumentPresentation = Extract< - SourceDocumentPresentation, - { readonly kind: "page-assets" } -> - -type CountChunksClient = { - readonly documents: { - listChunks( - documentId: string, - params: { readonly page: number; readonly pageSize: number }, - ): Promise<{ - readonly pagination?: { readonly total?: number } - }> - } - readonly knowledge: { - readChunks(params: { - readonly documentId: string - readonly revisionKey?: string - readonly chunkType: "page" - readonly page: number - readonly pageSize: number - readonly assetUrlPolicy: "durable" - }): Promise<{ - readonly chunks: readonly KnowledgeReadChunk[] - readonly totalChunks?: number - }> - } -} - -export type SourceViewOptionsLoadOptions = { - readonly documentPresentationDetection?: "enabled" | "disabled" - readonly getKnowledgeForSource?: (source: Source) => Knowledge -} - -export type SourceViewOptions = { - readonly chunkCount?: number - readonly documentPresentation?: SourceDocumentPresentation -} - -export const sourceViewOptionsBySourceId = ( - sources: readonly Source[], - client: Knowhere, - options: SourceViewOptionsLoadOptions = {}, -) => - Effect.gen(function* () { - const countClient = client as unknown as CountChunksClient - const readySources = sources.filter( - (source) => - !source.demoKey && - source.status === "ready" && - source.knowhereDocumentId, - ) - if (readySources.length === 0) return new Map() - - const entries = yield* Effect.all( - readySources.map((source) => - Effect.gen(function* () { - const loadedOptions = yield* Effect.tryPromise(() => - loadSourceViewOptions(countClient, source, options), - ).pipe( - Effect.catchAll(() => - Effect.sync((): SourceViewOptions | undefined => undefined), - ), - ) - return [source.id, loadedOptions] as const - }), - ), - { concurrency: "unbounded" }, - ) - - return new Map( - entries.filter( - (entry): entry is readonly [string, SourceViewOptions] => - entry[1] !== undefined, - ), - ) - }) - -export const countChunksBySourceId = ( - sources: readonly Source[], - client: Knowhere, -) => - Effect.gen(function* () { - const sourceOptions = yield* sourceViewOptionsBySourceId(sources, client) - const countEntries: [string, number][] = [] - for (const [sourceId, options] of sourceOptions.entries()) { - if (typeof options.chunkCount === "number") { - countEntries.push([sourceId, options.chunkCount]) - } - } - return new Map(countEntries) - }) - -async function loadSourceViewOptions( - client: CountChunksClient, - source: Source, - options: SourceViewOptionsLoadOptions, -): Promise { - const documentId = source.knowhereDocumentId - if (!documentId) return undefined - - if (options.documentPresentationDetection !== "disabled") { - const pagePresentation = await loadPageAssetPresentation( - client, - source, - options, - ) - if (pagePresentation) { - return { - chunkCount: pagePresentation.pageCount, - documentPresentation: pagePresentation, - } - } - } - - const chunkCount = await loadSourceChunkCount(client, documentId) - return typeof chunkCount === "number" ? { chunkCount } : undefined -} - -async function loadPageAssetPresentation( - client: CountChunksClient, - source: Source, - options: SourceViewOptionsLoadOptions, -): Promise { - const documentId = source.knowhereDocumentId - if (!documentId) return undefined - - try { - const knowledge = options.getKnowledgeForSource?.(source) ?? client.knowledge - const response = await knowledge.readChunks({ - documentId, - ...(source.knowhereJobId ? { revisionKey: source.knowhereJobId } : {}), - chunkType: "page", - page: 1, - pageSize: 1, - assetUrlPolicy: "durable", - }) - const firstChunk = response.chunks[0] - if (!firstChunk || firstChunk.chunkType !== "page") return undefined - const maxPageAssetNumber = getMaxUsablePageAssetNumber( - firstChunk.metadata.pageAssets, - ) - if (!maxPageAssetNumber) return undefined - - const totalChunks = getPositiveFiniteNumber(response.totalChunks) ?? 0 - const pageCount = Math.max(totalChunks, maxPageAssetNumber) - - return { kind: "page-assets", pageCount } - } catch { - return undefined - } -} - -async function loadSourceChunkCount( - client: CountChunksClient, - documentId: string, -): Promise { - const response = await client.documents.listChunks(documentId, { - page: 1, - pageSize: 1, - }) - const total = response.pagination?.total - return typeof total === "number" && Number.isFinite(total) ? total : undefined -} - -function getMaxUsablePageAssetNumber(value: unknown): number | undefined { - if (!Array.isArray(value)) return undefined - - const pageNumbers = value.flatMap((item): number[] => { - if (!isRecord(item)) return [] - const pageNum = item.pageNum - const artifactRef = item.artifactRef - const assetUrl = item.assetUrl - const isUsable = - typeof pageNum === "number" && - Number.isSafeInteger(pageNum) && - pageNum > 0 && - ((typeof artifactRef === "string" && artifactRef.trim().length > 0) || - (typeof assetUrl === "string" && assetUrl.trim().length > 0)) - return isUsable ? [pageNum] : [] - }) - if (pageNumbers.length === 0) return undefined - return Math.max(...pageNumbers) -} - -function getPositiveFiniteNumber(value: unknown): number | undefined { - return typeof value === "number" && Number.isFinite(value) && value > 0 - ? value - : undefined -} - -function isRecord(value: unknown): value is Readonly> { - return typeof value === "object" && value !== null -} diff --git a/src/domains/sources/demo-source-repository.ts b/src/domains/sources/demo-source-repository.ts index dea38ce6..71f92618 100644 --- a/src/domains/sources/demo-source-repository.ts +++ b/src/domains/sources/demo-source-repository.ts @@ -17,6 +17,7 @@ type UpsertMaterializedDemoSourceInput = { readonly sizeBytes: number readonly knowhereDocumentId: string readonly originalBlobUrl: string | null + readonly chunkCount?: number } type DemoSourceRepository = { @@ -102,6 +103,9 @@ const upsertMaterializedDemoSourceEffect: DemoSourceRepository["upsertMaterializ knowhereDocumentId: input.knowhereDocumentId, originalBlobUrl: input.originalBlobUrl, demoKey: input.demoSourceId, + ...(typeof input.chunkCount === "number" + ? { chunkCount: input.chunkCount } + : {}), }) .onConflictDoUpdate({ target: [sources.workspaceId, sources.demoKey], @@ -114,6 +118,9 @@ const upsertMaterializedDemoSourceEffect: DemoSourceRepository["upsertMaterializ knowhereJobId: null, knowhereDocumentId: input.knowhereDocumentId, originalBlobUrl: input.originalBlobUrl, + ...(typeof input.chunkCount === "number" + ? { chunkCount: input.chunkCount } + : {}), deletedAt: null, updatedAt: sql`now()`, }, diff --git a/src/domains/sources/reconcile.test.ts b/src/domains/sources/reconcile.test.ts index 5be0cead..9e255a0f 100644 --- a/src/domains/sources/reconcile.test.ts +++ b/src/domains/sources/reconcile.test.ts @@ -28,6 +28,7 @@ function makeSource(overrides: Partial): Source { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-06T00:00:00Z"), updatedAt: new Date("2026-05-06T00:00:00Z"), deletedAt: null, diff --git a/src/domains/sources/remote-library.test.ts b/src/domains/sources/remote-library.test.ts index 1a0df68d..fea50267 100644 --- a/src/domains/sources/remote-library.test.ts +++ b/src/domains/sources/remote-library.test.ts @@ -23,6 +23,7 @@ const localSource: Source = { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-10T00:00:00Z"), updatedAt: new Date("2026-05-10T00:00:00Z"), deletedAt: null, diff --git a/src/domains/sources/repository.ts b/src/domains/sources/repository.ts index f26f9f75..1e9a6087 100644 --- a/src/domains/sources/repository.ts +++ b/src/domains/sources/repository.ts @@ -16,6 +16,7 @@ type SourceRepository = { readonly markParsingEffect: typeof sourceRowRepository.markParsingEffect readonly markReadyEffect: typeof sourceRowRepository.markReadyEffect readonly updateRevisionKeyEffect: typeof sourceRowRepository.updateRevisionKeyEffect + readonly recordChunkCountEffect: typeof sourceRowRepository.recordChunkCountEffect readonly markFailedEffect: typeof sourceRowRepository.markFailedEffect readonly clearStagedBlobEffect: typeof sourceRowRepository.clearStagedBlobEffect readonly softDeleteEffect: typeof sourceRowRepository.softDeleteEffect @@ -42,6 +43,7 @@ export const sourceRepository: SourceRepository = { markParsingEffect: sourceRowRepository.markParsingEffect, markReadyEffect: sourceRowRepository.markReadyEffect, updateRevisionKeyEffect: sourceRowRepository.updateRevisionKeyEffect, + recordChunkCountEffect: sourceRowRepository.recordChunkCountEffect, markFailedEffect: sourceRowRepository.markFailedEffect, clearStagedBlobEffect: sourceRowRepository.clearStagedBlobEffect, softDeleteEffect: sourceRowRepository.softDeleteEffect, diff --git a/src/domains/sources/retry.test.ts b/src/domains/sources/retry.test.ts index 36c0eecc..c930dce5 100644 --- a/src/domains/sources/retry.test.ts +++ b/src/domains/sources/retry.test.ts @@ -134,6 +134,7 @@ function makeSource(overrides: Partial = {}): Source { originalBlobUrl: "https://store.public.blob.vercel-storage.com/source-uploads/upload_1/document.pdf", demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-10T00:00:00Z"), updatedAt: new Date("2026-05-10T00:00:00Z"), deletedAt: null, diff --git a/src/domains/sources/route-dependencies.ts b/src/domains/sources/route-dependencies.ts index f55c1316..04ceab9a 100644 --- a/src/domains/sources/route-dependencies.ts +++ b/src/domains/sources/route-dependencies.ts @@ -11,7 +11,6 @@ import { import { knowhereDemoApi } from "@/integrations/knowhere-demo" import { getCurrentUser, requireUser } from "@/infrastructure/auth" import { workspaceService } from "@/domains/workspace/service" -import { sourceViewOptionsBySourceId as getDefaultSourceViewOptionsBySourceId } from "./counts" import { createParsedDocumentSyncScheduler } from "./parsed-document-sync-scheduler" import { reconcileSourcesForWorkspace as reconcileDefaultSourcesForWorkspace } from "./reconcile" import { sourceWorkflowRuntime } from "./workflow-runtime" @@ -28,12 +27,6 @@ const defaultDependencies: SourceRouteServiceDependencies = { ensureApiKeyForWorkspace, ensureWorkspace: workspaceService.ensureWorkspace, getCurrentUser, - getSourceViewOptionsBySourceId: (sources, client, options) => - getDefaultSourceViewOptionsBySourceId( - sources, - client as ReturnType, - options, - ), makeKnowhereClient: (apiKey: string) => makeDefaultKnowhereClient(apiKey) as SourceRouteKnowhereClient, listSourcesForWorkspace: sourceWorkflowRuntime.listForWorkspace, diff --git a/src/domains/sources/route-listing.ts b/src/domains/sources/route-listing.ts index 08c17b26..ba6e699d 100644 --- a/src/domains/sources/route-listing.ts +++ b/src/domains/sources/route-listing.ts @@ -1,11 +1,7 @@ import { Effect } from "effect" import { demoView } from "@/domains/demo/view" -import { - getMaterializedDemoSourceViewOptionsBySourceId, - getWorkspaceSourcesNeedingChunkCount, - resolveWorkspaceDemoSources, -} from "@/domains/demo/workspace-source-resolution" +import { resolveWorkspaceDemoSources } from "@/domains/demo/workspace-source-resolution" import { routeResult } from "@/lib/route-result" import { logger } from "@/lib/logger" import { knowhereDemoApi } from "@/integrations/knowhere-demo" @@ -27,7 +23,6 @@ type RouteListingDependencies = Pick< | "ensureApiKeyForWorkspace" | "ensureWorkspace" | "getCurrentUser" - | "getSourceViewOptionsBySourceId" | "listSourcesForWorkspace" | "makeKnowhereClient" > & { @@ -96,10 +91,6 @@ const listSourcesEffect = ( client, localSources: demoSourceResolution.workspaceSources, }) - const sourcesNeedingChunkCount = - getWorkspaceSourcesNeedingChunkCount(workspaceSources) - const materializedDemoSourceOptions = - getMaterializedDemoSourceViewOptionsBySourceId(workspaceSources, catalog) yield* Effect.sync(() => triggerBackgroundReconciliationForParsingSources({ workspaceId: workspace.id, @@ -110,13 +101,6 @@ const listSourcesEffect = ( defaultStartBackgroundReconciliation, }), ) - const sourceOptions = yield* deps.getSourceViewOptionsBySourceId( - sourcesNeedingChunkCount, - client, - { - documentPresentationDetection: "disabled", - }, - ) const hiddenDemoSourceIds = new Set( yield* Effect.tryPromise(() => deps.sourceService.listHiddenDemoSourceIds(workspace.id), @@ -135,13 +119,7 @@ const listSourcesEffect = ( return routeResult.ok({ sources: [ ...visibleDemoSources, - ...workspaceSources.map((source) => - toSourceView( - source, - materializedDemoSourceOptions.get(source.id) ?? - sourceOptions.get(source.id), - ), - ), + ...workspaceSources.map((source) => toSourceView(source)), ...remoteSourceViews, ], }) diff --git a/src/domains/sources/route-retry.test.ts b/src/domains/sources/route-retry.test.ts index fc1316cc..5ecba810 100644 --- a/src/domains/sources/route-retry.test.ts +++ b/src/domains/sources/route-retry.test.ts @@ -27,6 +27,7 @@ function makeSource(overrides: Partial = {}): Source { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-10T00:00:00.000Z"), updatedAt: new Date("2026-05-10T00:00:00.000Z"), deletedAt: null, diff --git a/src/domains/sources/route-service.test.ts b/src/domains/sources/route-service.test.ts index 64c902af..35c72dc6 100644 --- a/src/domains/sources/route-service.test.ts +++ b/src/domains/sources/route-service.test.ts @@ -1,5 +1,4 @@ import { describe, expect, it, vi } from "vitest"; -import { Effect } from "effect"; import type { Job } from "@ontos-ai/knowhere-sdk"; import type { Source, Workspace } from "@/infrastructure/db/schema"; @@ -30,6 +29,7 @@ const source: Source = { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-10T00:00:00Z"), updatedAt: new Date("2026-05-10T00:00:00Z"), deletedAt: null, @@ -59,9 +59,6 @@ describe("source route service", () => { }, }; const ensureApiKeyForWorkspace = vi.fn(async () => "jwt_123"); - const getSourceViewOptionsBySourceId = vi.fn(() => - Effect.succeed(new Map([[source.id, { chunkCount: 8 }]])), - ); const listSourcesForWorkspace = vi.fn(async () => [source]); const reconcileSourcesForWorkspace = vi.fn(async () => [source]); const startBackgroundReconciliation = vi.fn(async () => undefined); @@ -77,7 +74,6 @@ describe("source route service", () => { email: null, name: null, })), - getSourceViewOptionsBySourceId, makeKnowhereClient: vi.fn(() => knowhereClient), listSourcesForWorkspace, reconcileSourcesForWorkspace, @@ -101,7 +97,6 @@ describe("source route service", () => { status: "parsing", mimeType: "application/pdf", documentId: undefined, - chunkCount: 8, }, ], }, @@ -286,7 +281,6 @@ describe("source route service", () => { email: null, name: null, })), - getSourceViewOptionsBySourceId: vi.fn(() => Effect.succeed(new Map())), makeKnowhereClient: vi.fn(() => knowhereClient), listSourcesForWorkspace: vi.fn(async () => [localReadySource]), reconcileSourcesForWorkspace: vi.fn(async () => [localReadySource]), @@ -408,7 +402,6 @@ describe("source route service", () => { email: null, name: null, })), - getSourceViewOptionsBySourceId: vi.fn(() => Effect.succeed(new Map())), makeKnowhereClient: vi.fn(() => knowhereClient), listSourcesForWorkspace: vi.fn(async () => [parsingSource]), reconcileSourcesForWorkspace, @@ -466,9 +459,6 @@ describe("source route service", () => { upload: vi.fn(), }, }; - const getSourceViewOptionsBySourceId = vi.fn(() => - Effect.succeed(new Map([[source.id, { chunkCount: 8 }]])), - ); const listing = createRouteListing({ demoApi: { fetchCatalog: vi.fn(async () => { @@ -482,7 +472,6 @@ describe("source route service", () => { email: null, name: null, })), - getSourceViewOptionsBySourceId, makeKnowhereClient: vi.fn(() => knowhereClient), listSourcesForWorkspace: vi.fn(async () => [legacyFakeSource, source]), reconcileSourcesForWorkspace: vi.fn(async () => [ @@ -497,13 +486,6 @@ describe("source route service", () => { const result = await listing.listSources({ cookieHeader: "session=abc" }); - expect(getSourceViewOptionsBySourceId).toHaveBeenCalledWith( - [source], - knowhereClient, - expect.objectContaining({ - documentPresentationDetection: "disabled", - }), - ); expect(result).toEqual({ status: 200, body: { @@ -515,7 +497,6 @@ describe("source route service", () => { status: "parsing", mimeType: "application/pdf", documentId: undefined, - chunkCount: 8, }, ], }, @@ -550,7 +531,6 @@ describe("source route service", () => { upload: vi.fn(), }, }; - const getSourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map())); const listing = createRouteListing({ demoApi: { fetchCatalog: vi.fn(async () => demoCatalog), @@ -562,7 +542,6 @@ describe("source route service", () => { email: null, name: null, })), - getSourceViewOptionsBySourceId, makeKnowhereClient: vi.fn(() => knowhereClient), listSourcesForWorkspace: vi.fn(async () => [legacyFakeSource]), reconcileSourcesForWorkspace: vi.fn(async () => [legacyFakeSource]), @@ -574,13 +553,6 @@ describe("source route service", () => { const result = await listing.listSources({ cookieHeader: "session=abc" }); - expect(getSourceViewOptionsBySourceId).toHaveBeenCalledWith( - [], - knowhereClient, - expect.objectContaining({ - documentPresentationDetection: "disabled", - }), - ); expect(result).toEqual({ status: 200, body: { @@ -635,7 +607,6 @@ describe("source route service", () => { upload: vi.fn(), }, }; - const getSourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map())); const listing = createRouteListing({ demoApi: { fetchCatalog: vi.fn(async () => demoCatalog), @@ -647,7 +618,6 @@ describe("source route service", () => { email: null, name: null, })), - getSourceViewOptionsBySourceId, makeKnowhereClient: vi.fn(() => knowhereClient), listSourcesForWorkspace: vi.fn(async () => [nonReadyLegacySource]), reconcileSourcesForWorkspace: vi.fn(async () => [nonReadyLegacySource]), @@ -659,13 +629,6 @@ describe("source route service", () => { const result = await listing.listSources({ cookieHeader: "session=abc" }); - expect(getSourceViewOptionsBySourceId).toHaveBeenCalledWith( - [], - knowhereClient, - expect.objectContaining({ - documentPresentationDetection: "disabled", - }), - ); expect(result).toEqual({ status: 200, body: { @@ -680,7 +643,7 @@ describe("source route service", () => { }); }); - it("uses demo catalog counts for materialized demo sources", async () => { + it("uses stored chunk counts for materialized demo sources", async () => { const materializedSource: Source = { ...source, id: "source_demo", @@ -690,6 +653,7 @@ describe("source route service", () => { knowhereJobId: null, knowhereDocumentId: "doc_user_copy", originalBlobUrl: "/api/demo-sources/demo-tsla-q4-2025/original", + chunkCount: 70, }; const knowhereClient = { documents: { @@ -710,7 +674,6 @@ describe("source route service", () => { upload: vi.fn(), }, }; - const getSourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map())); const listing = createRouteListing({ demoApi: { fetchCatalog: vi.fn(async () => demoCatalog), @@ -722,7 +685,6 @@ describe("source route service", () => { email: null, name: null, })), - getSourceViewOptionsBySourceId, makeKnowhereClient: vi.fn(() => knowhereClient), listSourcesForWorkspace: vi.fn(async () => [materializedSource]), reconcileSourcesForWorkspace: vi.fn(async () => [materializedSource]), @@ -734,13 +696,6 @@ describe("source route service", () => { const result = await listing.listSources({ cookieHeader: "session=abc" }); - expect(getSourceViewOptionsBySourceId).toHaveBeenCalledWith( - [], - knowhereClient, - expect.objectContaining({ - documentPresentationDetection: "disabled", - }), - ); expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled(); expect(result).toEqual({ status: 200, diff --git a/src/domains/sources/route-types.ts b/src/domains/sources/route-types.ts index 6787ae48..7907d315 100644 --- a/src/domains/sources/route-types.ts +++ b/src/domains/sources/route-types.ts @@ -16,10 +16,6 @@ import type { } from "@/integrations/knowhere-demo" import type { RouteResult } from "@/lib/route-result" import type { SourceBlobUploadInput } from "./blob-upload" -import type { - sourceViewOptionsBySourceId, - SourceViewOptionsLoadOptions, -} from "./counts" import type { UploadKnowhereClient } from "./upload" type SourceRouteKnowhereClient = UploadKnowhereClient & @@ -244,11 +240,6 @@ type SourceRouteServiceDependencies = { ) => Promise readonly ensureWorkspace: (userId: string) => Promise readonly getCurrentUser: () => Promise - readonly getSourceViewOptionsBySourceId: ( - sources: readonly Source[], - client: SourceRouteKnowhereClient, - options?: SourceViewOptionsLoadOptions, - ) => ReturnType readonly makeKnowhereClient: (apiKey: string) => SourceRouteKnowhereClient readonly listSourcesForWorkspace: (workspaceId: string) => Promise readonly reconcileSourcesForWorkspace: ( diff --git a/src/domains/sources/source-reconcile-route-workflow.test.ts b/src/domains/sources/source-reconcile-route-workflow.test.ts index 1498fed9..a622e4c5 100644 --- a/src/domains/sources/source-reconcile-route-workflow.test.ts +++ b/src/domains/sources/source-reconcile-route-workflow.test.ts @@ -6,6 +6,7 @@ const mocks = vi.hoisted(() => ({ releaseSyncCapacity: vi.fn(), updateSyncStatus: vi.fn(), updateRevisionKey: vi.fn(), + recordChunkCount: vi.fn(), markFailed: vi.fn(), loggerError: vi.fn(), loggerInfo: vi.fn(), @@ -31,6 +32,7 @@ vi.mock("@/domains/sources/workflow-runtime", () => ({ markFailed: mocks.markFailed, updateSyncStatus: mocks.updateSyncStatus, updateRevisionKey: mocks.updateRevisionKey, + recordChunkCount: mocks.recordChunkCount, }, })) @@ -78,6 +80,7 @@ function createClient(overrides: { const listChunks = vi.fn(async () => ({ jobResultId: overrides.jobResultId ?? "rev_1", jobId: overrides.jobId ?? "job_1", + pagination: { total: 12 }, })) return { client: { jobs: {}, documents: { listChunks } }, @@ -107,6 +110,7 @@ describe("sourceReconcileRouteWorkflow", () => { status: "ready", }) mocks.updateRevisionKey.mockResolvedValue({ id: "source_1" }) + mocks.recordChunkCount.mockResolvedValue({ id: "source_1", chunkCount: 12 }) mocks.withFreshKnowhereApiKey.mockImplementation( async (apiKey: string, run: (apiKey: string) => Promise) => ({ result: await run(apiKey), @@ -180,6 +184,11 @@ describe("sourceReconcileRouteWorkflow", () => { "source_1", "rev_1", ) + expect(mocks.recordChunkCount).toHaveBeenCalledWith( + "workspace_1", + "source_1", + 12, + ) expect(mocks.updateSyncStatus).toHaveBeenCalledWith( "workspace_1", "source_1", @@ -197,6 +206,39 @@ describe("sourceReconcileRouteWorkflow", () => { expect(continuations).toEqual([]) }) + it("does not record a chunk count when parse listChunks omits total", async () => { + const context = createWorkflowContext() + const listChunks = vi.fn(async () => ({ + jobResultId: "rev_1", + jobId: "job_1", + })) + mocks.makeKnowhereClientWithParsedStorage.mockReturnValue({ + client: { jobs: {}, documents: { listChunks } }, + knowledge: { syncParsedDocument: vi.fn() }, + }) + mocks.pollSourceReconciliation.mockResolvedValue({ + kind: "ready-to-prepare", + jobId: "job_1", + documentId: "doc_1", + }) + + await sourceReconcileRouteWorkflow.runPollAndMirrorWorkflow({ + context, + payload: sourceReconcileRouteWorkflow.normalizeReconcilePayload({ + workspaceId: "workspace_1", + sourceId: "source_1", + apiKey: "jwt_1", + }), + }) + + expect(mocks.updateRevisionKey).toHaveBeenCalledWith( + "workspace_1", + "source_1", + "rev_1", + ) + expect(mocks.recordChunkCount).not.toHaveBeenCalled() + }) + it("keeps the source ready when parsed-sync enqueue fails", async () => { const context = createWorkflowContext() const wired = createClient({}) diff --git a/src/domains/sources/source-reconcile-route-workflow.ts b/src/domains/sources/source-reconcile-route-workflow.ts index a1a316bc..7d1e7b39 100644 --- a/src/domains/sources/source-reconcile-route-workflow.ts +++ b/src/domains/sources/source-reconcile-route-workflow.ts @@ -53,6 +53,7 @@ type RevisionKeyClient = { ) => Promise<{ readonly jobResultId?: string | null readonly jobId?: string | null + readonly pagination?: { readonly total?: number } }> } } @@ -166,11 +167,21 @@ async function runPollAndMirrorWorkflow(input: { }), ) apiKey = revision.apiKey - const revisionKey = revision.result + const revisionKey = revision.result.revisionKey + const chunkCount = revision.result.chunkCount await context.run("record-source-revision-key", async () => sourceWorkflowRuntime.updateRevisionKey(workspaceId, sourceId, revisionKey), ) + if (typeof chunkCount === "number") { + await context.run("record-source-chunk-count", async () => + sourceWorkflowRuntime.recordChunkCount( + workspaceId, + sourceId, + chunkCount, + ), + ) + } await context.run("record-sync-pending", async () => sourceWorkflowRuntime.updateSyncStatus(workspaceId, sourceId, { revisionKey, @@ -202,14 +213,22 @@ async function resolveParsedRevisionKey(input: { readonly sourceId: string readonly documentId: string readonly fallbackRevisionKey: string -}): Promise { +}): Promise<{ + readonly revisionKey: string + readonly chunkCount?: number +}> { try { const firstPage = await input.client.documents.listChunks(input.documentId, { page: 1, pageSize: 1, includeAssetUrls: false, }) - return firstPage.jobResultId ?? firstPage.jobId ?? input.fallbackRevisionKey + const chunkCount = getRecordedChunkCount(firstPage.pagination?.total) + return { + revisionKey: + firstPage.jobResultId ?? firstPage.jobId ?? input.fallbackRevisionKey, + ...(chunkCount !== undefined ? { chunkCount } : {}), + } } catch (error) { logger.warn("workflow: failed to resolve parsed revision key", { sourceId: input.sourceId, @@ -217,10 +236,19 @@ async function resolveParsedRevisionKey(input: { fallbackRevisionKey: input.fallbackRevisionKey, error: getErrorMessage(error), }) - return input.fallbackRevisionKey + return { revisionKey: input.fallbackRevisionKey } } } +function getRecordedChunkCount(value: unknown): number | undefined { + return typeof value === "number" && + Number.isFinite(value) && + Number.isInteger(value) && + value >= 0 + ? value + : undefined +} + async function enqueueParsedSyncBestEffort(input: { readonly workspaceId: string readonly sourceId: string diff --git a/src/domains/sources/source-reconcile-workflow.test.ts b/src/domains/sources/source-reconcile-workflow.test.ts index 68769b70..f5aa7adf 100644 --- a/src/domains/sources/source-reconcile-workflow.test.ts +++ b/src/domains/sources/source-reconcile-workflow.test.ts @@ -32,6 +32,7 @@ function makeSource(overrides: Partial = {}): Source { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-06T00:00:00Z"), updatedAt: new Date("2026-05-06T00:00:00Z"), deletedAt: null, diff --git a/src/domains/sources/source-row-repository.test.ts b/src/domains/sources/source-row-repository.test.ts index 4b2833fb..7693cdef 100644 --- a/src/domains/sources/source-row-repository.test.ts +++ b/src/domains/sources/source-row-repository.test.ts @@ -98,6 +98,7 @@ async function captureLocalizeConflictSet(input: { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-06-26T00:00:00Z"), updatedAt: new Date("2026-06-26T00:00:00Z"), deletedAt: null, diff --git a/src/domains/sources/source-row-repository.ts b/src/domains/sources/source-row-repository.ts index 256158f2..d4fc778e 100644 --- a/src/domains/sources/source-row-repository.ts +++ b/src/domains/sources/source-row-repository.ts @@ -33,6 +33,7 @@ type SourceUpdate = Partial< | "stagedBlobUrl" | "originalBlobPathname" | "originalBlobUrl" + | "chunkCount" > > @@ -83,6 +84,11 @@ type SourceRowRepository = { sourceId: string, revisionKey: string, ) => Effect.Effect + readonly recordChunkCountEffect: ( + workspaceId: string, + sourceId: string, + chunkCount: number, + ) => Effect.Effect readonly markFailedEffect: ( workspaceId: string, sourceId: string, @@ -209,6 +215,7 @@ const markParsingEffect: SourceRowRepository["markParsingEffect"] = ( knowhereDocumentId: documentId, failureReason: null, failureStage: null, + chunkCount: null, }, requiredStatus) const markReadyEffect: SourceRowRepository["markReadyEffect"] = ( @@ -232,6 +239,15 @@ const updateRevisionKeyEffect: SourceRowRepository["updateRevisionKeyEffect"] = knowhereJobId: revisionKey, }, "ready") +const recordChunkCountEffect: SourceRowRepository["recordChunkCountEffect"] = ( + workspaceId: string, + sourceId: string, + chunkCount: number, +) => + updateInWorkspaceEffect(workspaceId, sourceId, { + chunkCount, + }, "ready") + const markFailedEffect: SourceRowRepository["markFailedEffect"] = ( workspaceId: string, sourceId: string, @@ -456,6 +472,7 @@ export const sourceRowRepository: SourceRowRepository = { markParsingEffect, markReadyEffect, updateRevisionKeyEffect, + recordChunkCountEffect, markFailedEffect, clearStagedBlobEffect, softDeleteEffect, diff --git a/src/domains/sources/upload.test.ts b/src/domains/sources/upload.test.ts index a9ad74c9..f217e5b4 100644 --- a/src/domains/sources/upload.test.ts +++ b/src/domains/sources/upload.test.ts @@ -31,6 +31,7 @@ function makeSource(overrides: Partial = {}): Source { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-06T00:00:00Z"), updatedAt: new Date("2026-05-06T00:00:00Z"), deletedAt: null, diff --git a/src/domains/sources/view.test.ts b/src/domains/sources/view.test.ts index 0069f6af..9da3e987 100644 --- a/src/domains/sources/view.test.ts +++ b/src/domains/sources/view.test.ts @@ -20,6 +20,7 @@ function makeSource(overrides: Partial = {}): Source { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-06T00:00:00Z"), updatedAt: new Date("2026-05-06T00:00:00Z"), deletedAt: null, @@ -35,8 +36,8 @@ describe("toSourceView", () => { originalBlobPathname: "source-uploads/upload_1/document.pdf", originalBlobUrl: "https://store.public.blob.vercel-storage.com/source-uploads/upload_1/document.pdf", + chunkCount: 7, }), - { chunkCount: 7 }, ), ).toEqual({ id: "source_1", @@ -105,8 +106,8 @@ describe("toSourceView", () => { sizeBytes: 5648867, knowhereDocumentId: "doc_user_copy", originalBlobUrl: "https://example.com/tsla-q4-2025.pdf", + chunkCount: 70, }), - { chunkCount: 70 }, ), ).toMatchObject({ title: "TSLA-Q4-2025-Update.pdf", diff --git a/src/domains/sources/view.ts b/src/domains/sources/view.ts index 5c6fe50d..bac67e00 100644 --- a/src/domains/sources/view.ts +++ b/src/domains/sources/view.ts @@ -20,6 +20,7 @@ export function toSourceView( } = {}, ): SourceView { const originalFile = getSourceOriginalFile(source) + const chunkCount = getStoredChunkCount(options.chunkCount ?? source.chunkCount) const status = toSourceStatus(source.status) const failureMessage = status === "failed" @@ -36,15 +37,17 @@ export function toSourceView( documentId: source.knowhereDocumentId ?? undefined, ...(failureMessage ? { failureMessage } : {}), ...(originalFile ? { originalFile } : {}), - ...(options.chunkCount !== undefined - ? { chunkCount: options.chunkCount } - : {}), + ...(chunkCount !== undefined ? { chunkCount } : {}), ...(options.documentPresentation !== undefined ? { documentPresentation: options.documentPresentation } : {}), }; } +function getStoredChunkCount(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) ? value : undefined +} + function toSourceStatus(status: string): SourceView["status"] { const result = Schema.decodeUnknownEither(SourceStatus)(status) if (result._tag === "Right") return result.right diff --git a/src/domains/sources/workflow-runtime.test.ts b/src/domains/sources/workflow-runtime.test.ts index 08e9853f..785e781e 100644 --- a/src/domains/sources/workflow-runtime.test.ts +++ b/src/domains/sources/workflow-runtime.test.ts @@ -39,6 +39,7 @@ function makeSource(status: Source["status"]): Source { originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-10T00:00:00.000Z"), updatedAt: new Date("2026-05-10T00:00:00.000Z"), deletedAt: null, diff --git a/src/domains/sources/workflow-runtime.ts b/src/domains/sources/workflow-runtime.ts index 9b0487ee..376dda62 100644 --- a/src/domains/sources/workflow-runtime.ts +++ b/src/domains/sources/workflow-runtime.ts @@ -95,6 +95,11 @@ type SourceWorkflowRuntime = UploadRepositoryRuntime & { sourceId: string, revisionKey: string, ) => Promise + readonly recordChunkCount: ( + workspaceId: string, + sourceId: string, + chunkCount: number, + ) => Promise readonly saveParseResult: ( workspaceId: string, sourceId: string, @@ -206,6 +211,19 @@ const updateRevisionKey: SourceWorkflowRuntime["updateRevisionKey"] = ( ), ) +const recordChunkCount: SourceWorkflowRuntime["recordChunkCount"] = ( + workspaceId: string, + sourceId: string, + chunkCount: number, +) => + databaseRuntime.runPromise( + sourceRepository.recordChunkCountEffect( + workspaceId, + sourceId, + chunkCount, + ), + ) + const markFailed: SourceWorkflowRuntime["markFailed"] = ( workspaceId: string, sourceId: string, @@ -341,6 +359,7 @@ export const sourceWorkflowRuntime: SourceWorkflowRuntime = { markParsing, markReady, updateRevisionKey, + recordChunkCount, mergeParseAssetUrls, saveParseResult, softDelete, diff --git a/src/domains/workspace/initial-state.test.ts b/src/domains/workspace/initial-state.test.ts index 0ed05e23..67d8b8a9 100644 --- a/src/domains/workspace/initial-state.test.ts +++ b/src/domains/workspace/initial-state.test.ts @@ -1,4 +1,3 @@ -import { Effect } from "effect" import { afterEach, describe, expect, it, vi } from "vitest" import { loadWorkspaceShellInitialState } from "./initial-state" @@ -114,14 +113,11 @@ describe("loadWorkspaceShellInitialState", () => { it("lists visible API demos before authenticated workspace sources", async () => { const workspace = makeWorkspace() - const source = makeSource(workspace.id) + const source = makeSource(workspace.id, { chunkCount: 2 }) const thread = makeThread(workspace.id) const deps = createDependencies({ listChatThreads: vi.fn(async () => [thread]), listSourcesForWorkspace: vi.fn(async () => [source]), - sourceViewOptionsBySourceId: vi.fn(() => - Effect.succeed(new Map([[source.id, { chunkCount: 2 }]])), - ), }) const state = await loadWorkspaceShellInitialState(deps) @@ -149,33 +145,22 @@ describe("loadWorkspaceShellInitialState", () => { it("keeps authenticated workspace sources when the demo catalog is unavailable", async () => { const workspace = makeWorkspace() - const source = makeSource(workspace.id) + const source = makeSource(workspace.id, { chunkCount: 2 }) const legacyFakeSource = makeSource(workspace.id, { id: "source_legacy_demo", demoKey: "demo-tsla-q4-2025", knowhereJobId: null, knowhereDocumentId: "demo-doc-tsla-q4-2025", }) - const sourceViewOptionsBySourceId = vi.fn(() => - Effect.succeed(new Map([[source.id, { chunkCount: 2 }]])), - ) const deps = createDependencies({ fetchDemoCatalog: vi.fn(async () => { throw new Error("Demo API unavailable.") }), listSourcesForWorkspace: vi.fn(async () => [legacyFakeSource, source]), - sourceViewOptionsBySourceId, }) const state = await loadWorkspaceShellInitialState(deps) - expect(sourceViewOptionsBySourceId).toHaveBeenCalledWith( - [source], - expect.any(Object), - expect.objectContaining({ - documentPresentationDetection: "disabled", - }), - ) expect(state.sources).toEqual([ { id: source.id, @@ -196,23 +181,15 @@ describe("loadWorkspaceShellInitialState", () => { demoKey: "demo-tsla-q4-2025", title: "TSLA-Q4-2025-Update.pdf", knowhereDocumentId: "doc_user_copy", + chunkCount: 70, }) - const sourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map())) const deps = createDependencies({ listHiddenDemoSourceIds: vi.fn(async () => ["another-demo"]), listSourcesForWorkspace: vi.fn(async () => [materializedSource]), - sourceViewOptionsBySourceId, }) const state = await loadWorkspaceShellInitialState(deps) - expect(sourceViewOptionsBySourceId).toHaveBeenCalledWith( - [], - expect.any(Object), - expect.objectContaining({ - documentPresentationDetection: "disabled", - }), - ) expect(state.sources).toEqual([ expect.objectContaining({ id: "source_demo", @@ -232,21 +209,12 @@ describe("loadWorkspaceShellInitialState", () => { knowhereJobId: null, knowhereDocumentId: "demo-doc-tsla-q4-2025", }) - const sourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map())) const deps = createDependencies({ listSourcesForWorkspace: vi.fn(async () => [legacyFakeSource]), - sourceViewOptionsBySourceId, }) const state = await loadWorkspaceShellInitialState(deps) - expect(sourceViewOptionsBySourceId).toHaveBeenCalledWith( - [], - expect.any(Object), - expect.objectContaining({ - documentPresentationDetection: "disabled", - }), - ) expect(state.sources).toEqual([ expect.objectContaining({ id: "demo-tsla-q4-2025", @@ -266,21 +234,12 @@ describe("loadWorkspaceShellInitialState", () => { knowhereJobId: null, knowhereDocumentId: null, }) - const sourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map())) const deps = createDependencies({ listSourcesForWorkspace: vi.fn(async () => [nonReadyLegacySource]), - sourceViewOptionsBySourceId, }) const state = await loadWorkspaceShellInitialState(deps) - expect(sourceViewOptionsBySourceId).toHaveBeenCalledWith( - [], - expect.any(Object), - expect.objectContaining({ - documentPresentationDetection: "disabled", - }), - ) expect(state.sources).toEqual([ expect.objectContaining({ id: "demo-tsla-q4-2025", @@ -509,26 +468,6 @@ describe("loadWorkspaceShellInitialState", () => { } }) - it("adds operation context when chunk-count lookup fails", async () => { - const deps = createDependencies({ - listSourcesForWorkspace: vi.fn(async () => [makeSource("workspace_1")]), - sourceViewOptionsBySourceId: vi.fn(() => - Effect.die(new Error("Knowhere document list timed out")), - ), - }) - - try { - await loadWorkspaceShellInitialState(deps) - throw new Error("Expected initial state loading to fail.") - } catch (error) { - const formatted = formatUnknownForLog(error) - - expect(formatted).toContain( - "Workspace initial state sourceViewOptionsBySourceId failed", - ) - expect(formatted).toContain("Knowhere document list timed out") - } - }) }) function createDependencies( @@ -553,7 +492,6 @@ function createDependencies( listMessages: vi.fn(async () => []), listSourcesForWorkspace: vi.fn(async () => []), reconcileSourcesForWorkspace: vi.fn(async () => []), - sourceViewOptionsBySourceId: vi.fn(() => Effect.succeed(new Map())), ...overrides, } } @@ -676,6 +614,7 @@ function makeSource( originalBlobPathname: null, originalBlobUrl: null, demoKey: null, + chunkCount: null, createdAt: new Date("2026-05-10T00:00:00.000Z"), updatedAt: new Date("2026-05-10T00:00:00.000Z"), deletedAt: null, diff --git a/src/domains/workspace/initial-state.ts b/src/domains/workspace/initial-state.ts index f959ad93..7e7e040e 100644 --- a/src/domains/workspace/initial-state.ts +++ b/src/domains/workspace/initial-state.ts @@ -5,14 +5,9 @@ import { Effect } from "effect" import type { ChatMessageView } from "@/domains/chat/types" import type { ParsedChunkView } from "@/domains/chunks/types" import { demoView } from "@/domains/demo/view" -import { - getMaterializedDemoSourceViewOptionsBySourceId, - getWorkspaceSourcesNeedingChunkCount, - resolveWorkspaceDemoSources, -} from "@/domains/demo/workspace-source-resolution" +import { resolveWorkspaceDemoSources } from "@/domains/demo/workspace-source-resolution" import { chatThreadService } from "@/domains/chat/thread-service" import { toChatMessageView, toChatThreadView } from "@/domains/chat/view" -import { sourceViewOptionsBySourceId as getSourceViewOptionsBySourceId } from "@/domains/sources/counts" import { listRemoteLibrarySourceViews } from "@/domains/sources/remote-library" import { reconcileSourcesForWorkspace as reconcileDefaultSourcesForWorkspace } from "@/domains/sources/reconcile" import { sourceService } from "@/domains/sources/service" @@ -93,7 +88,6 @@ async function getDemoChunksForSource( } type WorkspaceShellInitialStateClient = - Parameters[1] & Parameters[1] & { readonly documents: { readonly list: (params?: { @@ -146,11 +140,6 @@ type WorkspaceShellInitialStateDependencies = { client: WorkspaceShellInitialStateClient, ) => Promise readonly startBackgroundReconciliation?: typeof defaultStartBackgroundReconciliation - readonly sourceViewOptionsBySourceId: ( - sources: readonly Source[], - client: WorkspaceShellInitialStateClient, - options?: Parameters[2], - ) => ReturnType } const defaultDependencies: WorkspaceShellInitialStateDependencies = { @@ -165,7 +154,6 @@ const defaultDependencies: WorkspaceShellInitialStateDependencies = { listSourcesForWorkspace: sourceWorkflowRuntime.listForWorkspace, reconcileSourcesForWorkspace: reconcileDefaultSourcesForWorkspace, startBackgroundReconciliation: defaultStartBackgroundReconciliation, - sourceViewOptionsBySourceId: getSourceViewOptionsBySourceId, } // --------------------------------------------------------------------------- @@ -332,13 +320,6 @@ export const loadWorkspaceShellInitialStateEffect = ( localSources: demoSourceResolution.workspaceSources, }), ) - const sourcesNeedingChunkCount = - getWorkspaceSourcesNeedingChunkCount(workspaceSources) - const materializedDemoSourceOptions = - getMaterializedDemoSourceViewOptionsBySourceId( - workspaceSources, - demoCatalog, - ) yield* Effect.sync(() => triggerBackgroundReconciliationForParsingSources({ workspaceId: workspace.id, @@ -349,19 +330,6 @@ export const loadWorkspaceShellInitialStateEffect = ( defaultStartBackgroundReconciliation, }), ) - const sourceOptions = yield* effectOperation.addContext( - { - context: workspaceInitialStateContext, - operation: "sourceViewOptionsBySourceId", - }, - deps.sourceViewOptionsBySourceId( - sourcesNeedingChunkCount, - client, - { - documentPresentationDetection: "disabled", - }, - ), - ) return { user: { @@ -376,13 +344,7 @@ export const loadWorkspaceShellInitialStateEffect = ( dashboardUrl: resolveDashboardUrl(), sources: [ ...demoSources, - ...workspaceSources.map((source) => - toSourceView( - source, - materializedDemoSourceOptions.get(source.id) ?? - sourceOptions.get(source.id), - ), - ), + ...workspaceSources.map((source) => toSourceView(source)), ...remoteSourceViews, ], officialLibrarySources: toOfficialLibrarySourceViews(demoCatalog), diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index 4e9470f3..8a8466c7 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -2,6 +2,7 @@ import { sql } from "drizzle-orm"; import { bigint, index, + integer, jsonb, pgTable, text, @@ -78,6 +79,9 @@ export type NewWorkspace = typeof workspaces.$inferInsert; * older rows during the PR #28 transition * - `demo_key` — canonical demo source identifier when this row is a * materialized API-owned demo copy + * - `chunk_count` — Knowhere document total written when parse completes + * (or when a demo is materialized); homepage reads this + * locally and does not refetch chunks for the sidebar * - `deleted_at` — soft delete timestamp; reads filter it out * * Indexes: @@ -105,6 +109,7 @@ export const sources = pgTable( originalBlobPathname: text("original_blob_pathname"), originalBlobUrl: text("original_blob_url"), demoKey: text("demo_key"), + chunkCount: integer("chunk_count"), createdAt: timestamp("created_at", { withTimezone: true }) .notNull() .defaultNow(), From 63114917953224433b8e62775fe61e7973bf714b Mon Sep 17 00:00:00 2001 From: chengke <404835780@qq.com> Date: Fri, 11 Sep 2026 17:24:45 +0800 Subject: [PATCH 2/2] chore: drop local handover note from chunk-count change Keep the parse-complete total write and homepage count-fetch removal only. Co-authored-by: Cursor --- notebook-deferred-issues-2026-09-11.md | 36 -------------------------- 1 file changed, 36 deletions(-) delete mode 100644 notebook-deferred-issues-2026-09-11.md diff --git a/notebook-deferred-issues-2026-09-11.md b/notebook-deferred-issues-2026-09-11.md deleted file mode 100644 index ae8440a0..00000000 --- a/notebook-deferred-issues-2026-09-11.md +++ /dev/null @@ -1,36 +0,0 @@ -# Notebook 文档范围对齐记录（2026-09-11） - -## 原始问题 - -- 原始状态：代码链路已确认；未复现用户实际请求受影响。用户随后批准接通包含、排除两种范围，覆盖问题 2 的文档范围部分和问题 3。 -- Notebook 本地版本：`99cfe1d`；Knowhere 本地版本：`bf94dbe5`。 -- 实际路径：来源勾选状态 → `excludedSourceIds` → `excludeDocuments` 转为 `excludeDocumentIds` → Knowhere API 的 `exclude_document_ids` → retrieval context。 -- Knowhere 已接收参数。进入 agent_explore 时，`run_episode` 和 `ToolContext` 未传递该排除集合；最终 `assemble_retrieval_results` 才过滤结果。 -- 示例：排除 A 后，agent 仍可能读 A，最终 A 的证据被删除；是否实际发生、是否影响答案尚未验证。 -- 批准方案：将请求级包含、排除集合传到 Explore 工具上下文，各工具数据库查询在返回给模型前执行过滤，保留最终过滤；小语料和朴素检索执行同样范围。不增加模型调用，agent 在允许范围内继续自由探索。 -- 验证：排除 A 时，list/recall/grep/node_filter/read/assets/neighbors 均不能返回 A；未排除 B 仍能正常探索。空排除集合保持原行为。 -- 本次实现覆盖显式文档包含和排除；自然语言筛选仍由 agent 选择 node_filter 等工具执行。 - -代码位置： - -- Notebook：`src/components/workspace-chat-workflow.ts`、`src/domains/chat/retrieval.ts`、`src/domains/chat/index.ts`。 -- Knowhere：`apps/api/app/api/v1/routes/retrieval.py`、`packages/shared-python/shared/services/retrieval/execution/routes.py`、`packages/shared-python/shared/services/retrieval/agent_explore/dispatch.py`、`packages/shared-python/shared/services/retrieval/agent_tools/registry.py`。 - -## 参数约定与实现 - -- `includeDocumentIds` / `include_document_ids`：不传表示不限制，`[]` 表示空范围；只检索列表中的文档。 -- `excludeDocumentIds` / `exclude_document_ids`：排除列表中的文档；排除优先。 -- Notebook：现有 `knowhere_search` 工具增加两个字段；与资料排除状态合并。仅接受已有资料或本轮先前检索结果中确认的文档 ID，未知名称仍随自然语言 query 交给 Knowhere 定位。 -- SDK：新增包含参数类型与文档，HTTP 序列化保留空数组。Notebook 当前安装的 SDK 已用真实本机 HTTP 请求验证能正确发送，无需写入本地依赖路径。 -- Knowhere：统一 `DocumentScope` 作用于三种检索路径、八个 corpus 工具、最终引用和关联资源；缓存区分未限制、空集合、指定集合。 -- 清理仅限本次范围：Notebook 合并重复搜索请求类型；Knowhere 移除 recall 为限定文档而先查全库补集的旧转换。 -- 不涉及 DeepSeek 收工门禁、模型配置、路径/阈值语义或前端展示机制。 - -## 验证记录 - -- Notebook：聊天、账本、引用和媒体相关 193 项测试通过；TypeScript、ESLint、diff 检查通过。 -- SDK：61 项测试通过，包含两种认证方式下的实际 HTTP 字段检查；类型、lint 和格式检查通过。 -- Knowhere：42 项真实 PostgreSQL 合约测试、27 项共享测试通过；最后兼容性调整后，15 项范围合约测试再次通过。Pyright、Ruff、diff 检查通过。 -- 两种 Explore harness 使用真实执行循环和数据库工具，模型 provider 使用模拟回复；未调用线上模型，未验证其自然语言选择行为。数据库测试使用隔离测试库。 -- 尚未提交、发布或部署；线上需后端发布后才会执行新增范围语义。 -- Effect 指南：已查阅 `basics`、`testing`；保留现有 Effect 执行方式。