diff --git a/src/app/api/sources/[sourceId]/route.test.ts b/src/app/api/sources/[sourceId]/route.test.ts index 6f825747..f531c232 100644 --- a/src/app/api/sources/[sourceId]/route.test.ts +++ b/src/app/api/sources/[sourceId]/route.test.ts @@ -10,6 +10,7 @@ const mocks = vi.hoisted(() => { ensureWorkspace: vi.fn(), fetchDemoCatalog: vi.fn(), findSourceInWorkspace: vi.fn(), + findByKnowhereDocumentId: vi.fn(), getCurrentUser: vi.fn(), hideDemoSource: vi.fn(), makeKnowhereClient: vi.fn(), @@ -55,6 +56,7 @@ vi.mock("@/domains/sources/background-reconcile", () => ({ vi.mock("@/domains/sources/service", () => ({ sourceService: { findInWorkspace: mocks.findSourceInWorkspace, + findByKnowhereDocumentId: mocks.findByKnowhereDocumentId, hideDemoSource: mocks.hideDemoSource, retrySourceToKnowhere: mocks.retrySourceToKnowhere, softDelete: mocks.softDeleteSource, @@ -116,11 +118,17 @@ describe("PATCH /api/sources/[sourceId]", () => { ); }); - it("rejects archive requests for unlocalized remote source ids", async () => { + it("archives unlocalized remote Knowhere documents", async () => { mocks.requireUser.mockResolvedValue({ id: "user_1" }); mocks.ensureWorkspace.mockResolvedValue({ id: "workspace_1" }); mocks.findSourceInWorkspace.mockResolvedValue(null); + mocks.findByKnowhereDocumentId.mockResolvedValue(null); mocks.fetchDemoCatalog.mockResolvedValue({ sources: [] }); + mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123"); + mocks.makeKnowhereClient.mockReturnValue({ + documents: { archive: mocks.archive }, + }); + mocks.archive.mockResolvedValue(undefined); const response = await PATCH( new NextRequest( @@ -138,18 +146,71 @@ describe("PATCH /api/sources/[sourceId]", () => { ); await expect(response.json()).resolves.toEqual({ - message: "Source not found.", + id: "knowhere-doc:default:doc_remote", + archived: true, }); - expect(response.status).toBe(404); + expect(response.status).toBe(200); expect(mocks.findSourceInWorkspace).toHaveBeenCalledWith( "workspace_1", "knowhere-doc:default:doc_remote", ); - expect(mocks.archive).not.toHaveBeenCalled(); + expect(mocks.archive).toHaveBeenCalledWith("doc_remote"); + expect(mocks.findByKnowhereDocumentId).toHaveBeenCalledWith( + "workspace_1", + "doc_remote", + ); expect(mocks.softDeleteSource).not.toHaveBeenCalled(); expect(mocks.deleteBlob).not.toHaveBeenCalled(); }); + it("soft-deletes a matching local row when archiving a remote source id", async () => { + mocks.requireUser.mockResolvedValue({ id: "user_1" }); + mocks.ensureWorkspace.mockResolvedValue({ id: "workspace_1" }); + mocks.findSourceInWorkspace.mockResolvedValue(null); + mocks.findByKnowhereDocumentId.mockResolvedValue({ + id: "source_1", + knowhereDocumentId: "doc_remote", + originalBlobPathname: "source-uploads/upload_1/document.pdf", + demoKey: null, + }); + mocks.fetchDemoCatalog.mockResolvedValue({ sources: [] }); + mocks.ensureApiKeyForWorkspace.mockResolvedValue("jwt_123"); + mocks.makeKnowhereClient.mockReturnValue({ + documents: { archive: mocks.archive }, + }); + mocks.archive.mockResolvedValue(undefined); + mocks.softDeleteSource.mockResolvedValue(true); + + const response = await PATCH( + new NextRequest( + "http://localhost:3001/api/sources/knowhere-doc:default:doc_remote", + { + method: "PATCH", + body: JSON.stringify({ archived: true }), + }, + ), + { + params: Promise.resolve({ + sourceId: "knowhere-doc:default:doc_remote", + }), + }, + ); + + await expect(response.json()).resolves.toEqual({ + id: "knowhere-doc:default:doc_remote", + archived: true, + }); + expect(response.status).toBe(200); + expect(mocks.archive).toHaveBeenCalledWith("doc_remote"); + expect(mocks.softDeleteSource).toHaveBeenCalledWith( + "workspace_1", + "source_1", + ); + expect(mocks.deleteBlob).toHaveBeenCalledWith( + "source-uploads/upload_1/document.pdf", + ); + }); + it("does not fail an already-soft-deleted source when original Blob cleanup fails", async () => { mocks.requireUser.mockResolvedValue({ id: "user_1" }); mocks.ensureWorkspace.mockResolvedValue({ id: "workspace_1" }); diff --git a/src/components/chunks-panel-state.test.ts b/src/components/chunks-panel-state.test.ts index ae75c710..febfd71e 100644 --- a/src/components/chunks-panel-state.test.ts +++ b/src/components/chunks-panel-state.test.ts @@ -149,7 +149,7 @@ describe("chunksPanelState", () => { ]) }) - it("deduplicates page-asset chunks by page number", () => { + it("deduplicates singleton page-asset chunks with the same page number", () => { const chunks: ParsedChunkView[] = [ { chunkId: "page_4_first", @@ -208,6 +208,92 @@ describe("chunksPanelState", () => { ).toEqual(["page_4_first", "page_5"]) }) + it("keeps overlapping page-memory section chunks that share a boundary page", () => { + const chunks: ParsedChunkView[] = [ + { + chunkId: "kenneth", + type: "page", + content: "IR introduction.", + sectionPath: "call.pdf/Root/Kenneth Dorell", + sourceTitle: "call.pdf", + pageNums: [1], + pageAssets: [ + { + pageNumber: 1, + assetUrl: "https://assets.example/page-1.png", + contentType: "image/png", + }, + ], + }, + { + chunkId: "zuckerberg", + type: "page", + content: "[SAME-AS call.pdf/Root/Kenneth Dorell p1] CEO remarks.", + sectionPath: "call.pdf/Root/Mark Zuckerberg, CEO", + sourceTitle: "call.pdf", + pageNums: [1, 2], + pageAssets: [ + { + pageNumber: 1, + assetUrl: "https://assets.example/page-1.png", + contentType: "image/png", + }, + { + pageNumber: 2, + assetUrl: "https://assets.example/page-2.png", + contentType: "image/png", + }, + ], + }, + { + chunkId: "outlook", + type: "page", + content: "Q2 outlook.", + sectionPath: "call.pdf/Root/Moving to our financial outlook.", + sourceTitle: "call.pdf", + pageNums: [8], + pageAssets: [ + { + pageNumber: 8, + assetUrl: "https://assets.example/page-8.png", + contentType: "image/png", + }, + ], + }, + { + chunkId: "capex", + type: "page", + content: "[SAME-AS call.pdf/Root/Moving to our financial outlook. p8] Q&A.", + sectionPath: "call.pdf/Root/Turning to the expense and capex outlooks.", + sourceTitle: "call.pdf", + pageNums: [8, 9, 10], + pageAssets: [ + { + pageNumber: 8, + assetUrl: "https://assets.example/page-8.png", + contentType: "image/png", + }, + { + pageNumber: 9, + assetUrl: "https://assets.example/page-9.png", + contentType: "image/png", + }, + { + pageNumber: 10, + assetUrl: "https://assets.example/page-10.png", + contentType: "image/png", + }, + ], + }, + ] + + expect( + chunksPanelState + .getPageAssetChunksWithoutDuplicatePages(chunks) + .map((chunk) => chunk.chunkId), + ).toEqual(["kenneth", "zuckerberg", "outlook", "capex"]) + }) + it("hides table asset chunks from page-asset lists", () => { const chunks: ParsedChunkView[] = [ { diff --git a/src/components/chunks-panel-state.ts b/src/components/chunks-panel-state.ts index f11e855b..3afccbf9 100644 --- a/src/components/chunks-panel-state.ts +++ b/src/components/chunks-panel-state.ts @@ -192,29 +192,42 @@ function dedupeChunksById( function getPageAssetChunksWithoutDuplicatePages( chunks: readonly ParsedChunkView[], ): readonly ParsedChunkView[] { - const seenPageNumbers = new Set() + const seenSingletonPageNumbers = new Set() return chunks.filter((chunk) => { // Page-memory table assets currently store a file path, not HTML. if (chunk.type === "table") return false if (chunk.type !== "page") return true - const pageNumber = getPageAssetChunkPageNumber(chunk) - if (pageNumber === null) return true - if (seenPageNumbers.has(pageNumber)) return false + const pageNumbers = getPageAssetChunkPageNumbers(chunk) + if (pageNumbers.length === 0) return true + if (pageNumbers.length > 1) return true - seenPageNumbers.add(pageNumber) + const pageNumber = pageNumbers[0]! + if (seenSingletonPageNumbers.has(pageNumber)) return false + + seenSingletonPageNumbers.add(pageNumber) return true }) } -function getPageAssetChunkPageNumber(chunk: ParsedChunkView): number | null { - const pageAssetNumbers = (chunk.pageAssets ?? []) - .map((pageAsset) => pageAsset.pageNumber) - .filter(isPositivePageNumber) - if (pageAssetNumbers.length > 0) return Math.min(...pageAssetNumbers) +function getPageAssetChunkPageNumbers( + chunk: ParsedChunkView, +): readonly number[] { + const pageAssetNumbers = uniquePositivePageNumbers( + (chunk.pageAssets ?? []).map((pageAsset) => pageAsset.pageNumber), + ) + if (pageAssetNumbers.length > 0) return pageAssetNumbers + + return uniquePositivePageNumbers(chunk.pageNums ?? []) +} - return getFirstPageNumber(chunk) +function uniquePositivePageNumbers( + pageNumbers: readonly number[], +): readonly number[] { + return [...new Set(pageNumbers.filter(isPositivePageNumber))].sort( + (left, right) => left - right, + ) } function createMutableSectionTreeNode(input: { diff --git a/src/components/chunks-panel.test.ts b/src/components/chunks-panel.test.ts index 33367f1b..a56da3f7 100644 --- a/src/components/chunks-panel.test.ts +++ b/src/components/chunks-panel.test.ts @@ -198,6 +198,65 @@ describe("ChunksPanel", () => { .toBeNull(); }); + it("keeps overlapping page-memory sections that share a first page image", async () => { + mockVisibleVirtualViewport(); + + render( + React.createElement(C, { + chunks: [ + { + chunkId: "kenneth", + type: "page", + content: "IR introduction", + sectionPath: "call.pdf/Root/Kenneth Dorell", + sourceTitle: "call.pdf", + pageNums: [1], + pageAssets: [ + { + pageNumber: 1, + assetUrl: "https://assets.example/page-1.png", + contentType: "image/png", + }, + ], + }, + { + chunkId: "zuckerberg", + type: "page", + content: "CEO remarks", + sectionPath: "call.pdf/Root/Mark Zuckerberg, CEO", + sourceTitle: "call.pdf", + pageNums: [1, 2], + pageAssets: [ + { + pageNumber: 1, + assetUrl: "https://assets.example/page-1.png", + contentType: "image/png", + }, + { + pageNumber: 2, + assetUrl: "https://assets.example/page-2.png", + contentType: "image/png", + }, + ], + }, + ], + selectedSource: "call.pdf", + selectedSourceView: { + id: "source_1", + title: "call.pdf", + mimeType: "application/pdf", + status: "ready", + documentPresentation: { kind: "page-assets", pageCount: 2 }, + }, + }), + ); + + selectListView(); + expect(await screen.findByTestId("chunk-card-shell-kenneth")).toBeTruthy(); + expect(screen.getByTestId("chunk-card-shell-zuckerberg")).toBeTruthy(); + expect(screen.getByRole("img", { name: "Page 2" })).toBeTruthy(); + }); + it("renders page chunks normally when no page assets exist", async () => { mockVisibleVirtualViewport(); diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index e9556ac3..e26a6700 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -186,6 +186,15 @@ describe("answerQuestionWithRetrieval", () => { status: "ready", currentJobResultId: "job_remote", sourceFileName: "remote.pdf", + documentMetadata: { + createdByClient: "cli", + }, + }, + { + documentId: "doc_untagged", + namespace: "default", + status: "ready", + sourceFileName: "dummy.pdf", }, ], }); diff --git a/src/domains/chat/knowhere-tools.ts b/src/domains/chat/knowhere-tools.ts index 210b4870..5c70fd7b 100644 --- a/src/domains/chat/knowhere-tools.ts +++ b/src/domains/chat/knowhere-tools.ts @@ -8,6 +8,7 @@ import type { import type { Source } from "@/infrastructure/db/schema" import { listRemoteLibraryDocuments, + isNotebookVisibleRemoteDocument, type RemoteLibraryDocument, } from "@/domains/sources/remote-library" import type { SearchSources } from "./contracts" @@ -128,6 +129,7 @@ async function listVisibleRemoteDocuments(input: { return documents .filter( (document) => + isNotebookVisibleRemoteDocument(document) && document.status === "ready" && !localDocumentIds.has(document.documentId) && !input.excludedDocumentIds.has(document.documentId), diff --git a/src/domains/sources/document-metadata.test.ts b/src/domains/sources/document-metadata.test.ts index ba643462..6c69ac6d 100644 --- a/src/domains/sources/document-metadata.test.ts +++ b/src/domains/sources/document-metadata.test.ts @@ -2,7 +2,11 @@ import { describe, expect, it } from "vitest" import { createNotebookDocumentMetadata, + getCreatedByClient, + isNotebookVisibleRemoteClient, + isNotebookVisibleRemoteMetadata, NOTEBOOK_DOCUMENT_METADATA_DEFAULTS, + NOTEBOOK_VISIBLE_CREATED_BY_CLIENTS, } from "./document-metadata" describe("createNotebookDocumentMetadata", () => { @@ -46,4 +50,32 @@ describe("createNotebookDocumentMetadata", () => { clientVersion: "0.1.0", }) }) + + it("reads camelCase and snake_case created-by-client metadata", () => { + expect(getCreatedByClient({ createdByClient: "cli" })).toBe("cli") + expect(getCreatedByClient({ created_by_client: "mcp" })).toBe("mcp") + expect(getCreatedByClient({})).toBeUndefined() + expect(getCreatedByClient(undefined)).toBeUndefined() + }) + + it("allowlists notebook, cli, and mcp remote documents", () => { + expect(NOTEBOOK_VISIBLE_CREATED_BY_CLIENTS).toEqual([ + "notebook", + "cli", + "mcp", + ]) + expect(isNotebookVisibleRemoteClient("notebook")).toBe(true) + expect(isNotebookVisibleRemoteClient("cli")).toBe(true) + expect(isNotebookVisibleRemoteClient("mcp")).toBe(true) + expect(isNotebookVisibleRemoteClient("node-sdk")).toBe(false) + expect(isNotebookVisibleRemoteClient("api")).toBe(false) + expect(isNotebookVisibleRemoteClient(undefined)).toBe(false) + expect(isNotebookVisibleRemoteMetadata({ createdByClient: "cli" })).toBe( + true, + ) + expect(isNotebookVisibleRemoteMetadata({ created_by_client: "api" })).toBe( + false, + ) + expect(isNotebookVisibleRemoteMetadata({})).toBe(false) + }) }) diff --git a/src/domains/sources/document-metadata.ts b/src/domains/sources/document-metadata.ts index d6b608ff..b2be4d34 100644 --- a/src/domains/sources/document-metadata.ts +++ b/src/domains/sources/document-metadata.ts @@ -9,6 +9,19 @@ export const NOTEBOOK_DOCUMENT_METADATA_DEFAULTS = { clientVersion: packageJson.version, } as const; +export const NOTEBOOK_VISIBLE_CREATED_BY_CLIENTS = [ + "notebook", + "cli", + "mcp", +] as const; + +export type NotebookVisibleCreatedByClient = + (typeof NOTEBOOK_VISIBLE_CREATED_BY_CLIENTS)[number]; + +const notebookVisibleCreatedByClientSet = new Set( + NOTEBOOK_VISIBLE_CREATED_BY_CLIENTS, +); + export function createNotebookDocumentMetadata(input: { readonly title: string; readonly mimeType: string; @@ -24,3 +37,25 @@ export function createNotebookDocumentMetadata(input: { ...input.overrides, }; } + +export function getCreatedByClient( + metadata: Readonly> | undefined, +): string | undefined { + if (!metadata) return undefined; + const value = metadata.createdByClient ?? metadata.created_by_client; + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +export function isNotebookVisibleRemoteClient( + client: string | undefined, +): client is NotebookVisibleCreatedByClient { + return ( + client !== undefined && notebookVisibleCreatedByClientSet.has(client) + ); +} + +export function isNotebookVisibleRemoteMetadata( + metadata: Readonly> | undefined, +): boolean { + return isNotebookVisibleRemoteClient(getCreatedByClient(metadata)); +} diff --git a/src/domains/sources/remote-library.test.ts b/src/domains/sources/remote-library.test.ts new file mode 100644 index 00000000..1a0df68d --- /dev/null +++ b/src/domains/sources/remote-library.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it, vi } from "vitest" +import { Effect } from "effect" + +import type { Source } from "@/infrastructure/db/schema" +import { + isNotebookVisibleRemoteDocument, + listRemoteLibrarySourceViews, +} from "./remote-library" + +const localSource: Source = { + id: "source_local", + workspaceId: "workspace_1", + title: "notes.pdf", + mimeType: "application/pdf", + sizeBytes: 5, + status: "ready", + failureReason: null, + failureStage: null, + knowhereJobId: null, + knowhereDocumentId: "doc_local", + stagedBlobPathname: null, + stagedBlobUrl: null, + originalBlobPathname: null, + originalBlobUrl: null, + demoKey: null, + createdAt: new Date("2026-05-10T00:00:00Z"), + updatedAt: new Date("2026-05-10T00:00:00Z"), + deletedAt: null, +} + +describe("listRemoteLibrarySourceViews", () => { + it("keeps notebook, cli, and mcp remotes and hides other origins", async () => { + const list = vi.fn( + async (params?: { readonly namespace?: string }) => { + if (params?.namespace !== "default") { + return { documents: [] } + } + + return { + documents: [ + { + documentId: "doc_cli", + namespace: "default", + status: "active", + sourceFileName: "cli.pdf", + documentMetadata: { createdByClient: "cli" }, + }, + { + documentId: "doc_mcp", + namespace: "default", + status: "active", + sourceFileName: "mcp.pdf", + documentMetadata: { created_by_client: "mcp" }, + }, + { + documentId: "doc_notebook", + namespace: "default", + status: "active", + sourceFileName: "notebook.pdf", + documentMetadata: { createdByClient: "notebook" }, + }, + { + documentId: "doc_untagged", + namespace: "default", + status: "active", + sourceFileName: "dummy.pdf", + }, + { + documentId: "doc_sdk", + namespace: "default", + status: "active", + sourceFileName: "sdk.pdf", + documentMetadata: { createdByClient: "node-sdk" }, + }, + { + documentId: "doc_api", + namespace: "default", + status: "active", + sourceFileName: "api.pdf", + documentMetadata: { created_by_client: "api" }, + }, + { + documentId: "doc_local", + namespace: "default", + status: "active", + sourceFileName: "notes.pdf", + documentMetadata: { createdByClient: "cli" }, + }, + ], + } + }) + + const views = await Effect.runPromise( + listRemoteLibrarySourceViews({ + workspace: { namespace: "notebook-workspace_1" }, + client: { documents: { list } }, + localSources: [localSource], + }), + ) + + expect(views.map((view) => view.documentId)).toEqual([ + "doc_cli", + "doc_mcp", + "doc_notebook", + ]) + expect( + isNotebookVisibleRemoteDocument({ + documentMetadata: { createdByClient: "cli" }, + }), + ).toBe(true) + expect( + isNotebookVisibleRemoteDocument({ + documentMetadata: {}, + }), + ).toBe(false) + }) +}) diff --git a/src/domains/sources/remote-library.ts b/src/domains/sources/remote-library.ts index ceb9c807..77746d18 100644 --- a/src/domains/sources/remote-library.ts +++ b/src/domains/sources/remote-library.ts @@ -3,6 +3,10 @@ import { Effect } from "effect" import type { Source } from "@/infrastructure/db/schema" import type { SourceView } from "./types" import type { SourceStatus } from "./types" +import { + getCreatedByClient, + isNotebookVisibleRemoteMetadata, +} from "./document-metadata" import { getCompatibleNamespaces, sharedLibraryNamespace } from "./namespace" type RemoteDocument = { @@ -152,6 +156,7 @@ export function listRemoteLibrarySourceViews( ) const remoteDocuments = (yield* listRemoteLibraryDocuments(input)).filter( (document) => + isNotebookVisibleRemoteDocument(document) && !localDocumentIds.has(document.documentId) && !matchesActiveNotebookParsingSource(document, input.localSources), ) @@ -271,6 +276,12 @@ function normalizeRemoteDocument( } } +export function isNotebookVisibleRemoteDocument( + document: Pick, +): boolean { + return isNotebookVisibleRemoteMetadata(document.documentMetadata) +} + function toRemoteSourceView(document: RemoteDocument): SourceView { return { id: encodeRemoteSourceId(document), @@ -317,7 +328,7 @@ function matchesActiveNotebookParsingSource( document: RemoteDocument, localSources: readonly Source[], ): boolean { - if (document.documentMetadata?.createdByClient !== "notebook") return false + if (getCreatedByClient(document.documentMetadata) !== "notebook") return false if (!document.title || !document.mimeType || document.sizeBytes === undefined) { return false } diff --git a/src/domains/sources/repository.ts b/src/domains/sources/repository.ts index 3c6d01c3..f26f9f75 100644 --- a/src/domains/sources/repository.ts +++ b/src/domains/sources/repository.ts @@ -6,6 +6,7 @@ import { sourceRowRepository } from "./source-row-repository" type SourceRepository = { readonly findInWorkspaceEffect: typeof sourceRowRepository.findInWorkspaceEffect + readonly findByKnowhereDocumentIdEffect: typeof sourceRowRepository.findByKnowhereDocumentIdEffect readonly listForWorkspaceEffect: typeof sourceRowRepository.listForWorkspaceEffect readonly createUploadingEffect: typeof sourceRowRepository.createUploadingEffect readonly localizeRemoteDocumentEffect: typeof sourceRowRepository.localizeRemoteDocumentEffect @@ -28,6 +29,8 @@ type SourceRepository = { export const sourceRepository: SourceRepository = { findInWorkspaceEffect: sourceRowRepository.findInWorkspaceEffect, + findByKnowhereDocumentIdEffect: + sourceRowRepository.findByKnowhereDocumentIdEffect, listForWorkspaceEffect: sourceRowRepository.listForWorkspaceEffect, createUploadingEffect: sourceRowRepository.createUploadingEffect, localizeRemoteDocumentEffect: diff --git a/src/domains/sources/route-archive.ts b/src/domains/sources/route-archive.ts index 60472569..dc968ece 100644 --- a/src/domains/sources/route-archive.ts +++ b/src/domains/sources/route-archive.ts @@ -1,7 +1,9 @@ import { Effect } from "effect" +import type { Source } from "@/infrastructure/db/schema" import { routeResult } from "@/lib/route-result" import { getClientForWorkspace } from "./route-dependencies" +import { decodeRemoteSourceId } from "./remote-library" import type { ArchiveSourceBody, ArchiveSourceInput, @@ -51,38 +53,83 @@ const archiveSourceEffect = ( deps.sourceService.findInWorkspace(workspace.id, input.sourceId), ) - if (!source) { - const catalog = yield* Effect.tryPromise(() => deps.demoApi.fetchCatalog()) - const isDemoSource = catalog.sources.some( - (candidate) => candidate.demoSourceId === input.sourceId, - ) - if (isDemoSource) { - yield* Effect.tryPromise(() => - deps.sourceService.hideDemoSource(workspace.id, input.sourceId), + if (source) { + if (source.knowhereDocumentId) { + yield* archiveKnowhereDocument( + workspace.id, + input.cookieHeader, + source.knowhereDocumentId, + deps, ) - return routeResult.ok({ id: input.sourceId, archived: true as const }) } - return routeResult.error(404, "Source not found.") + yield* cleanupLocalSource(workspace.id, source, deps) + return routeResult.ok({ id: input.sourceId, archived: true as const }) } - if (source.knowhereDocumentId) { - const client = yield* Effect.tryPromise(() => - getClientForWorkspace(workspace.id, input.cookieHeader, deps), - ) + const catalog = yield* Effect.tryPromise(() => deps.demoApi.fetchCatalog()) + const isDemoSource = catalog.sources.some( + (candidate) => candidate.demoSourceId === input.sourceId, + ) + if (isDemoSource) { yield* Effect.tryPromise(() => - client.documents.archive(source.knowhereDocumentId!), - ).pipe( - Effect.catchIf(isKnowhereDocumentNotFoundError, () => Effect.void), + deps.sourceService.hideDemoSource(workspace.id, input.sourceId), ) + return routeResult.ok({ id: input.sourceId, archived: true as const }) + } + + const remoteSource = decodeRemoteSourceId(input.sourceId) + if (!remoteSource) { + return routeResult.error(404, "Source not found.") } + yield* archiveKnowhereDocument( + workspace.id, + input.cookieHeader, + remoteSource.documentId, + deps, + ) + + const localSource = yield* Effect.tryPromise(() => + deps.sourceService.findByKnowhereDocumentId( + workspace.id, + remoteSource.documentId, + ), + ) + if (localSource) { + yield* cleanupLocalSource(workspace.id, localSource, deps) + } + + return routeResult.ok({ id: input.sourceId, archived: true as const }) + }) + +const archiveKnowhereDocument = ( + workspaceId: string, + cookieHeader: string, + documentId: string, + deps: RouteArchiveDependencies, +) => + Effect.gen(function* () { + const client = yield* Effect.tryPromise(() => + getClientForWorkspace(workspaceId, cookieHeader, deps), + ) + yield* Effect.tryPromise(() => client.documents.archive(documentId)).pipe( + Effect.catchIf(isKnowhereDocumentNotFoundError, () => Effect.void), + ) + }) + +const cleanupLocalSource = ( + workspaceId: string, + source: Source, + deps: RouteArchiveDependencies, +) => + Effect.gen(function* () { yield* Effect.tryPromise(() => - deps.sourceService.softDelete(workspace.id, input.sourceId), + deps.sourceService.softDelete(workspaceId, source.id), ) if (source.demoKey) { yield* Effect.tryPromise(() => - deps.sourceService.hideDemoSource(workspace.id, source.demoKey!), + deps.sourceService.hideDemoSource(workspaceId, source.demoKey!), ) } if (source.originalBlobPathname) { @@ -90,8 +137,6 @@ const archiveSourceEffect = ( deps.deleteBlob(source.originalBlobPathname!), ).pipe(Effect.catchAllCause(() => Effect.void)) } - - return routeResult.ok({ id: input.sourceId, archived: true as const }) }) function isKnowhereDocumentNotFoundError(error: unknown): boolean { diff --git a/src/domains/sources/route-dependencies.ts b/src/domains/sources/route-dependencies.ts index b0cc8876..f55c1316 100644 --- a/src/domains/sources/route-dependencies.ts +++ b/src/domains/sources/route-dependencies.ts @@ -45,6 +45,7 @@ const defaultDependencies: SourceRouteServiceDependencies = { requireUser, sourceService: { findInWorkspace: defaultSourceService.findInWorkspace, + findByKnowhereDocumentId: defaultSourceService.findByKnowhereDocumentId, hideDemoSource: defaultSourceService.hideDemoSource, listHiddenDemoSourceIds: defaultSourceService.listHiddenDemoSourceIds, localizeRemoteDocument: defaultSourceService.localizeRemoteDocument, diff --git a/src/domains/sources/route-service.test.ts b/src/domains/sources/route-service.test.ts index cb9c7dae..64c902af 100644 --- a/src/domains/sources/route-service.test.ts +++ b/src/domains/sources/route-service.test.ts @@ -183,9 +183,34 @@ describe("source route service", () => { status: "active", sourceFileName: "cli.pdf", documentMetadata: { + createdByClient: "cli", mimeType: "application/pdf", }, }, + { + documentId: "doc_untagged", + namespace: "default", + status: "active", + sourceFileName: "dummy.pdf", + }, + { + documentId: "doc_sdk", + namespace: "default", + status: "active", + sourceFileName: "sdk.pdf", + documentMetadata: { + createdByClient: "node-sdk", + }, + }, + { + documentId: "doc_api", + namespace: "default", + status: "active", + sourceFileName: "api.pdf", + documentMetadata: { + created_by_client: "api", + }, + }, ], pagination: { page: 1, @@ -217,6 +242,9 @@ describe("source route service", () => { namespace: workspace.namespace, status: "active", sourceFileName: "legacy.pdf", + documentMetadata: { + createdByClient: "mcp", + }, }, ], pagination: { diff --git a/src/domains/sources/route-types.ts b/src/domains/sources/route-types.ts index 01d18830..6787ae48 100644 --- a/src/domains/sources/route-types.ts +++ b/src/domains/sources/route-types.ts @@ -188,6 +188,10 @@ type SourceWorkflowService = { workspaceId: string, sourceId: string, ) => Promise + readonly findByKnowhereDocumentId: ( + workspaceId: string, + documentId: string, + ) => Promise readonly softDelete: ( workspaceId: string, sourceId: string, diff --git a/src/domains/sources/service.ts b/src/domains/sources/service.ts index 4fcbfd58..1e4801cb 100644 --- a/src/domains/sources/service.ts +++ b/src/domains/sources/service.ts @@ -17,6 +17,10 @@ type SourceService = { workspaceId: string, sourceId: string, ) => Promise + readonly findByKnowhereDocumentId: ( + workspaceId: string, + documentId: string, + ) => Promise readonly listForWorkspace: (workspaceId: string) => Promise readonly localizeRemoteDocument: ( workspaceId: string, @@ -96,6 +100,7 @@ const retrySourceToKnowhere: SourceService["retrySourceToKnowhere"] = ( export const sourceService: SourceService = { findInWorkspace: sourceWorkflowRuntime.findInWorkspace, + findByKnowhereDocumentId: sourceWorkflowRuntime.findByKnowhereDocumentId, hideDemoSource: sourceWorkflowRuntime.hideDemoSource, listHiddenDemoSourceIds: sourceWorkflowRuntime.listHiddenDemoSourceIds, listForWorkspace: sourceWorkflowRuntime.listForWorkspace, diff --git a/src/domains/sources/source-row-repository.ts b/src/domains/sources/source-row-repository.ts index 2c06765a..256158f2 100644 --- a/src/domains/sources/source-row-repository.ts +++ b/src/domains/sources/source-row-repository.ts @@ -51,6 +51,10 @@ type SourceRowRepository = { workspaceId: string, sourceId: string, ) => Effect.Effect + readonly findByKnowhereDocumentIdEffect: ( + workspaceId: string, + documentId: string, + ) => Effect.Effect readonly listForWorkspaceEffect: ( workspaceId: string, ) => Effect.Effect @@ -126,6 +130,15 @@ const findInWorkspaceEffect: SourceRowRepository["findInWorkspaceEffect"] = ( ) }) +const findByKnowhereDocumentIdEffect: SourceRowRepository["findByKnowhereDocumentIdEffect"] = + (workspaceId: string, documentId: string) => + Effect.gen(function* () { + const db = yield* DbClient + return yield* Effect.promise(() => + findByKnowhereDocumentIdWithDb(db, workspaceId, documentId), + ) + }) + const listForWorkspaceEffect: SourceRowRepository["listForWorkspaceEffect"] = ( workspaceId: string, ) => @@ -301,6 +314,28 @@ async function findInWorkspaceWithDb( return row[0] ?? null } +async function findByKnowhereDocumentIdWithDb( + db: Db, + workspaceId: string, + documentId: string, +): Promise { + if (documentId.length === 0) return null + + const row = await db + .select() + .from(sources) + .where( + and( + eq(sources.workspaceId, workspaceId), + eq(sources.knowhereDocumentId, documentId), + isNull(sources.deletedAt), + ), + ) + .limit(1) + + return row[0] ?? null +} + async function updateInWorkspaceWithDb( db: Db, workspaceId: string, @@ -414,6 +449,7 @@ function requireSource(source: Source | null, message: string): Source { export const sourceRowRepository: SourceRowRepository = { findInWorkspaceEffect, + findByKnowhereDocumentIdEffect, listForWorkspaceEffect, createUploadingEffect, localizeRemoteDocumentEffect, diff --git a/src/domains/sources/workflow-runtime.ts b/src/domains/sources/workflow-runtime.ts index 24464e01..9b0487ee 100644 --- a/src/domains/sources/workflow-runtime.ts +++ b/src/domains/sources/workflow-runtime.ts @@ -59,6 +59,10 @@ type SourceWorkflowRuntime = UploadRepositoryRuntime & { workspaceId: string, sourceId: string, ) => Promise + readonly findByKnowhereDocumentId: ( + workspaceId: string, + documentId: string, + ) => Promise readonly getParseAssetUrls: ( workspaceId: string, sourceId: string, @@ -124,6 +128,12 @@ const findInWorkspace: SourceWorkflowRuntime["findInWorkspace"] = ( sourceRepository.findInWorkspaceEffect(workspaceId, sourceId), ) +const findByKnowhereDocumentId: SourceWorkflowRuntime["findByKnowhereDocumentId"] = + (workspaceId: string, documentId: string) => + databaseRuntime.runPromise( + sourceRepository.findByKnowhereDocumentIdEffect(workspaceId, documentId), + ) + const listForWorkspace: SourceWorkflowRuntime["listForWorkspace"] = ( workspaceId: string, ) => @@ -319,6 +329,7 @@ export const sourceWorkflowRuntime: SourceWorkflowRuntime = { createUploadRepository, createUploading, findInWorkspace, + findByKnowhereDocumentId, getParseAssetUrls, getParseResultProgress, getParseSnapshotMetadata,