diff --git a/.env.local.example b/.env.local.example index 272d031..f75dd2a 100644 --- a/.env.local.example +++ b/.env.local.example @@ -2,6 +2,11 @@ # Use staging when validating staging keys. # KNOWHERE_BASE_URL=https://api-staging.knowhereto.ai +# Optional development override. When set, Notebook skips Dashboard session +# auth and Dashboard-issued JWT creation, then calls Knowhere directly with +# this key. Leave unset for production and Dashboard-authenticated staging. +# KNOWHERE_API_KEY=sk_your_development_key_here + # --- Chat provider (server-side only) --- # Vercel AI Gateway key; AI SDK picks it up automatically AI_GATEWAY_API_KEY=vck_your_key_here diff --git a/README.md b/README.md index a0111b2..5e4e239 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Upload documents, explore parsed content, and ask questions about your knowledge 2. Fill in your API keys in `.env.local`: - `AI_GATEWAY_API_KEY` โ€” your Vercel AI Gateway key for chat (optional `CHAT_MODEL` override) + - `KNOWHERE_API_KEY` โ€” optional development override that skips Dashboard auth and calls Knowhere directly 3. Install dependencies and run: ```bash @@ -42,6 +43,12 @@ Notebook treats Dashboard as the auth source of truth. Server-side auth calls forward the incoming session cookie to Dashboard oRPC endpoints, including `/api/orpc/users/getCurrentUser` and `/api/orpc/users/issueServiceJwt`. +For local development, setting server-side `KNOWHERE_API_KEY` switches Notebook +into API-key mode. In that mode the app uses a deterministic local development +user, skips Dashboard redirects and JWT issuance, and passes the configured key +directly to the Knowhere SDK. Leave it unset for production and normal +Dashboard-authenticated staging flows. + Dashboard chooses its oRPC handler by request shape and `Content-Type`. When using Effect's `HttpClientRequest.bodyText`, pass `"application/json"` as the body content type. Setting the header before diff --git a/drizzle.config.ts b/drizzle.config.ts index 7148eb7..03d9850 100644 --- a/drizzle.config.ts +++ b/drizzle.config.ts @@ -9,7 +9,7 @@ import { defineConfig } from "drizzle-kit"; * pnpm db:migrate # apply migrations to DATABASE_URL (prod deploy) */ export default defineConfig({ - schema: "./src/lib/schema.ts", + schema: "./src/infrastructure/db/schema.ts", out: "./drizzle", dialect: "postgresql", dbCredentials: { diff --git a/package.json b/package.json index bb9d756..6570d03 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "db:generate": "drizzle-kit generate", "db:push": "drizzle-kit push", "db:migrate": "drizzle-kit migrate", - "db:studio": "drizzle-kit studio" + "db:studio": "drizzle-kit studio", + "upstash:dev": "npx @upstash/qstash-cli dev" }, "dependencies": { "@ai-sdk/react": "^3.0.177", @@ -34,6 +35,7 @@ "@radix-ui/react-tabs": "^1.1.13", "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-virtual": "^3.13.24", + "@upstash/workflow": "^1.2.1", "@vercel/blob": "^2.3.3", "ai": "^6.0.175", "class-variance-authority": "^0.7.1", @@ -79,4 +81,4 @@ "typescript": "^6.0.3", "vitest": "^4.1.5" } -} +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 303e312..16c3536 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -53,6 +53,9 @@ importers: '@tanstack/react-virtual': specifier: ^3.13.24 version: 3.13.24(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@upstash/workflow': + specifier: ^1.2.1 + version: 1.2.1(zod@4.4.3) '@vercel/blob': specifier: ^2.3.3 version: 2.3.3 @@ -2311,6 +2314,14 @@ packages: cpu: [x64] os: [win32] + '@upstash/qstash@2.11.0': + resolution: {integrity: sha512-AfPPxsUeOJCrxMQ9dkh1RZL40wgxCsUNFkxrbBSomC3U4j4qKFRawU8sDK+dqqH+sZFUS1biVM4GK46d5Tg2Vg==} + + '@upstash/workflow@1.2.1': + resolution: {integrity: sha512-G2WfWruKXbPpKJNyCWS097jtRZXW993BeMPYJPc7gGoEjnpJTH9S9j/YsfRJXZxWBPAuHB31tMzmKMGWAkAMQw==} + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + '@vercel/blob@2.3.3': resolution: {integrity: sha512-MtD7VLo6hU07eHR7bmk5SIMD290q574UaNYTe46qeyRT+hWrCy26CoAqfd7PnIefVXvRehRZBzukxuTO9iGTVg==} engines: {node: '>=20.0.0'} @@ -2705,6 +2716,9 @@ packages: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} + crypto-js@4.2.0: + resolution: {integrity: sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==} + css-tree@3.2.1: resolution: {integrity: sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==} engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0} @@ -3750,6 +3764,9 @@ packages: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true + jose@5.10.0: + resolution: {integrity: sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==} + jose@6.2.3: resolution: {integrity: sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==} @@ -4211,6 +4228,10 @@ packages: resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} engines: {node: '>= 0.6'} + neverthrow@7.2.0: + resolution: {integrity: sha512-iGBUfFB7yPczHHtA8dksKTJ9E8TESNTAx1UQWW6TzMF280vo9jdPYpLUXrMN1BCkPdHFdNG3fxOt2CUad8KhAw==} + engines: {node: '>=18'} + next-themes@0.4.6: resolution: {integrity: sha512-pZvgD5L0IEvX5/9GWyHMf3m8BKiVQwsCMHfoFosXtXBMnaS0ZnIJ9ST4b4NqLVKDEm8QBxoNNGNaBv2JNF6XNA==} peerDependencies: @@ -7104,6 +7125,17 @@ snapshots: '@unrs/resolver-binding-win32-x64-msvc@1.11.1': optional: true + '@upstash/qstash@2.11.0': + dependencies: + crypto-js: 4.2.0 + jose: 5.10.0 + neverthrow: 7.2.0 + + '@upstash/workflow@1.2.1(zod@4.4.3)': + dependencies: + '@upstash/qstash': 2.11.0 + zod: 4.4.3 + '@vercel/blob@2.3.3': dependencies: async-retry: 1.3.3 @@ -7508,6 +7540,8 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 + crypto-js@4.2.0: {} + css-tree@3.2.1: dependencies: mdn-data: 2.27.1 @@ -8671,6 +8705,8 @@ snapshots: jiti@2.7.0: {} + jose@5.10.0: {} + jose@6.2.3: {} js-tokens@4.0.0: {} @@ -9325,6 +9361,8 @@ snapshots: negotiator@1.0.0: {} + neverthrow@7.2.0: {} + next-themes@0.4.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4): dependencies: react: 19.2.4 diff --git a/src/app/api/demo-sources/materialize/route.ts b/src/app/api/demo-sources/materialize/route.ts index 273e82c..8e6f4b6 100644 --- a/src/app/api/demo-sources/materialize/route.ts +++ b/src/app/api/demo-sources/materialize/route.ts @@ -1,5 +1,11 @@ +import { Effect } from "effect" import type { NextResponse } from "next/server" +import { chatCitationPersistence } from "@/domains/chat/chat-citation-persistence" +import { chatMessageRepository } from "@/domains/chat/chat-message-repository" +import { chatThreadRepository } from "@/domains/chat/chat-thread-repository" +import type { ChatCitationView } from "@/domains/chat/types" +import { databaseRuntime } from "@/domains/workspace/database-runtime" import { sourceService } from "@/domains/sources/service" import { toSourceView } from "@/domains/sources/view" import { notebookRequestContext } from "@/domains/workspace/request-context" @@ -8,72 +14,101 @@ import { nextRouteResponse } from "@/lib/next-route-response" import { routeResult } from "@/lib/route-result" export async function POST(request: Request): Promise { - const body = await routeResult.readJson(request) - if (!body.ok) { - return nextRouteResponse.toNextResponse( - routeResult.badRequest("Invalid request body."), - ) - } + return Effect.runPromise( + Effect.gen(function* () { + const body = yield* Effect.tryPromise(() => + routeResult.readJson(request), + ) + if (!body.ok) { + return nextRouteResponse.toNextResponse( + routeResult.badRequest("Invalid request body."), + ) + } - const demoSourceIds = getDemoSourceIds(body.value) - if (demoSourceIds.length === 0) { - return nextRouteResponse.toNextResponse( - routeResult.badRequest("Select at least one demo source."), - ) - } + const demoSourceIds = getDemoSourceIds(body.value) + if (demoSourceIds.length === 0) { + return nextRouteResponse.toNextResponse( + routeResult.badRequest("Select at least one demo source."), + ) + } - try { - const { apiKey, workspace } = - await notebookRequestContext.getAuthenticatedWithClient() - const hiddenDemoSourceIds = new Set( - await sourceService.listHiddenDemoSourceIds(workspace.id), - ) - const visibleDemoSourceIds = demoSourceIds.filter( - (demoSourceId) => !hiddenDemoSourceIds.has(demoSourceId), - ) - if (visibleDemoSourceIds.length === 0) { - return nextRouteResponse.toNextResponse( - routeResult.badRequest("Selected demo sources are no longer available."), + const { apiKey, workspace } = yield* Effect.tryPromise(() => + notebookRequestContext.getAuthenticatedWithClient(), ) - } - - const materializedSources = await knowhereDemoApi.materializeSources({ - apiKey, - namespace: workspace.namespace, - demoSourceIds: visibleDemoSourceIds, - }) - const sources = await Promise.all( - materializedSources.map(async (source) => { - const row = await sourceService.upsertMaterializedDemoSource( - workspace.id, - { - demoSourceId: source.demoSourceId, - title: source.title, - mimeType: source.mimeType, - sizeBytes: source.sizeBytes, - knowhereDocumentId: source.documentId, - originalBlobUrl: `/api/demo-sources/${encodeURIComponent( - source.demoSourceId, - )}/original`, - }, + const hiddenDemoSourceIds = new Set( + yield* Effect.tryPromise(() => + sourceService.listHiddenDemoSourceIds(workspace.id), + ), + ) + const visibleDemoSourceIds = demoSourceIds.filter( + (demoSourceId) => !hiddenDemoSourceIds.has(demoSourceId), + ) + if (visibleDemoSourceIds.length === 0) { + return nextRouteResponse.toNextResponse( + routeResult.badRequest( + "Selected demo sources are no longer available.", + ), ) - return toSourceView(row, { chunkCount: source.chunkCount }) - }), - ) - - return nextRouteResponse.toNextResponse(routeResult.ok({ sources })) - } catch { - return nextRouteResponse.toNextResponse( - routeResult.error(502, "Demo sources could not be prepared right now."), - ) - } + } + + const materializedSources = yield* Effect.tryPromise(() => + knowhereDemoApi.materializeSources({ + apiKey, + namespace: workspace.namespace, + demoSourceIds: visibleDemoSourceIds, + }), + ) + + const sources = yield* Effect.all( + materializedSources.map((source) => + Effect.gen(function* () { + const row = yield* Effect.tryPromise(() => + sourceService.upsertMaterializedDemoSource(workspace.id, { + demoSourceId: source.demoSourceId, + title: source.title, + mimeType: source.mimeType, + sizeBytes: source.sizeBytes, + knowhereDocumentId: source.documentId, + originalBlobUrl: `/api/demo-sources/${encodeURIComponent( + source.demoSourceId, + )}/original`, + }), + ) + return toSourceView(row, { chunkCount: source.chunkCount }) + }), + ), + { concurrency: "unbounded" }, + ) + + // After materialization, remap seeded demo-thread citations from their + // canonical document IDs to the new materialized document IDs so source + // citation resolution continues to work. + yield* Effect.tryPromise(() => + fixDemoThreadCitations(workspace.id, materializedSources), + ).pipe(Effect.catchAllCause(() => Effect.void)) + + return nextRouteResponse.toNextResponse(routeResult.ok({ sources })) + }).pipe( + Effect.catchAll(() => + Effect.succeed( + nextRouteResponse.toNextResponse( + routeResult.error( + 502, + "Demo sources could not be prepared right now.", + ), + ), + ), + ), + ), + ) } function getDemoSourceIds(value: unknown): string[] { if (!isRecord(value) || !Array.isArray(value.demoSourceIds)) return [] const selectedIds = value.demoSourceIds.filter( - (item): item is string => typeof item === "string" && item.trim().length > 0, + (item): item is string => + typeof item === "string" && item.trim().length > 0, ) return Array.from(new Set(selectedIds.map((item) => item.trim()))) } @@ -81,3 +116,60 @@ function getDemoSourceIds(value: unknown): string[] { function isRecord(value: unknown): value is Readonly> { return typeof value === "object" && value !== null } + +const seededDemoChatKey = "knowhere-demo-chat" + +async function fixDemoThreadCitations( + workspaceId: string, + materializedSources: ReadonlyArray<{ + readonly demoSourceId: string + readonly documentId: string + }>, +): Promise { + const catalog = await knowhereDemoApi.fetchCatalog() + const canonicalIdByDemoSourceId = new Map( + catalog.sources.map((s) => [s.demoSourceId, s.canonicalDocumentId]), + ) + const documentIdMap = new Map() + for (const source of materializedSources) { + const canonical = canonicalIdByDemoSourceId.get(source.demoSourceId) + if (canonical) { + documentIdMap.set(canonical, source.documentId) + } + } + if (documentIdMap.size === 0) return + + const thread = await databaseRuntime.runPromise( + chatThreadRepository.findThreadByDemoKeyEffect( + workspaceId, + seededDemoChatKey, + ), + ) + if (!thread) return + + const messages = await databaseRuntime.runPromise( + chatMessageRepository.listMessagesForThreadEffect(workspaceId, thread.id), + ) + if (!messages || messages.length === 0) return + + await Promise.all( + messages.map(async (message) => { + const currentCitations = message.citations as + | ChatCitationView[] + | null + | undefined + const updated = chatCitationPersistence.replaceDemoCitationDocumentId( + currentCitations ?? undefined, + documentIdMap, + ) + if (!updated) return + + await databaseRuntime.runPromise( + chatMessageRepository.updateMessageCitationsEffect( + message.id, + chatCitationPersistence.normalizeCitations(updated), + ), + ) + }), + ) +} diff --git a/src/app/api/source-uploads/blob/route.ts b/src/app/api/source-uploads/blob/route.ts index 2dd5c73..a2c1d92 100644 --- a/src/app/api/source-uploads/blob/route.ts +++ b/src/app/api/source-uploads/blob/route.ts @@ -1,3 +1,4 @@ +import { Effect } from "effect" import { del } from "@vercel/blob" import { handleUpload, type HandleUploadBody } from "@vercel/blob/client" import type { NextRequest, NextResponse } from "next/server" @@ -13,81 +14,107 @@ import { nextRouteResponse } from "@/lib/next-route-response" import { routeResult } from "@/lib/route-result" export async function POST(request: NextRequest): Promise { - const user = await getCurrentUser() - if (!user) { - return nextRouteResponse.toNextResponse( - routeResult.error(401, "Please log in to upload documents."), - ) - } + return Effect.runPromise( + Effect.gen(function* () { + const user = yield* Effect.tryPromise(() => getCurrentUser()) + if (!user) { + return nextRouteResponse.toNextResponse( + routeResult.error(401, "Please log in to upload documents."), + ) + } - try { - const body = (await request.json()) as HandleUploadBody - const response = await handleUpload({ - body, - request, - onBeforeGenerateToken: async (pathname, clientPayload) => { - const input = parseSourceBlobClientPayload(clientPayload) - if (!input) { - throw new Error("Invalid upload metadata.") - } + const body = (yield* Effect.tryPromise(() => + request.json(), + )) as HandleUploadBody - const validation = validateSourceBlobUploadMetadata({ - ...input, - pathname, - }) - if (!validation.ok) { - throw new Error(validation.message) - } + const result = yield* Effect.tryPromise(() => + handleUpload({ + body, + request, + onBeforeGenerateToken: async (pathname, clientPayload) => { + const input = parseSourceBlobClientPayload(clientPayload) + if (!input) { + throw new Error("Invalid upload metadata.") + } - return { - addRandomSuffix: true, - allowOverwrite: false, - maximumSizeInBytes: MAX_UPLOAD_BYTES, - tokenPayload: JSON.stringify({ - userId: user.id, - fileName: validation.title, - mimeType: validation.mimeType, - sizeBytes: input.sizeBytes, - }), - } - }, - }) + const validation = validateSourceBlobUploadMetadata({ + ...input, + pathname, + }) + if (!validation.ok) { + throw new Error(validation.message) + } - return nextRouteResponse.toNextResponse(routeResult.ok(response)) - } catch (error) { - const message = error instanceof Error - ? error.message - : "Could not prepare the upload." - return nextRouteResponse.toNextResponse(routeResult.badRequest(message)) - } + return { + addRandomSuffix: true, + allowOverwrite: false, + maximumSizeInBytes: MAX_UPLOAD_BYTES, + tokenPayload: JSON.stringify({ + userId: user.id, + fileName: validation.title, + mimeType: validation.mimeType, + sizeBytes: input.sizeBytes, + }), + } + }, + }), + ).pipe( + Effect.map((response) => + nextRouteResponse.toNextResponse(routeResult.ok(response)), + ), + Effect.catchAll((error) => { + const message = + error instanceof Error + ? error.message + : "Could not prepare the upload." + return Effect.succeed( + nextRouteResponse.toNextResponse(routeResult.badRequest(message)), + ) + }), + ) + + return result + }), + ) } export async function DELETE(request: NextRequest): Promise { - const user = await getCurrentUser() - if (!user) { - return nextRouteResponse.toNextResponse( - routeResult.error(401, "Please log in to upload documents."), - ) - } + return Effect.runPromise( + Effect.gen(function* () { + const user = yield* Effect.tryPromise(() => getCurrentUser()) + if (!user) { + return nextRouteResponse.toNextResponse( + routeResult.error(401, "Please log in to upload documents."), + ) + } - try { - const body = (await request.json()) as unknown - const pathname = getCleanupPathname(body) - if (!pathname || !isValidSourceBlobPathname(pathname)) { - return nextRouteResponse.toNextResponse( - routeResult.badRequest( - "Invalid upload path. Choose the document again.", + const body = (yield* Effect.tryPromise(() => + request.json(), + ).pipe( + Effect.catchAll( + (): Effect.Effect => Effect.succeed(null), ), - ) - } + )) as unknown + + const pathname = getCleanupPathname(body) + if (!pathname || !isValidSourceBlobPathname(pathname)) { + return nextRouteResponse.toNextResponse( + routeResult.badRequest("Invalid upload path. Choose the document again."), + ) + } - await del(pathname) - return nextRouteResponse.toNextResponse(routeResult.ok({ ok: true })) - } catch { - return nextRouteResponse.toNextResponse( - routeResult.error(500, "Could not clean up the upload."), - ) - } + yield* Effect.tryPromise(() => del(pathname)) + return nextRouteResponse.toNextResponse(routeResult.ok({ ok: true })) + }).pipe( + Effect.catchAll(() => + Effect.succeed( + nextRouteResponse.toNextResponse( + routeResult.error(500, "Could not clean up the upload."), + ), + ), + ), + ), + ) } function getCleanupPathname(body: unknown): string | null { diff --git a/src/app/api/sources/[sourceId]/chunks/route.test.ts b/src/app/api/sources/[sourceId]/chunks/route.test.ts index 4629b41..4b7ebf7 100644 --- a/src/app/api/sources/[sourceId]/chunks/route.test.ts +++ b/src/app/api/sources/[sourceId]/chunks/route.test.ts @@ -199,6 +199,218 @@ describe("GET /api/sources/[sourceId]/chunks", () => { }) }) + it("serves API-owned demo chunks for authenticated canonical demo sources", async () => { + mocks.getCurrentUser.mockResolvedValue({ + id: "knowhere-api-key-dev-user", + email: null, + name: "Knowhere API Key Development", + }) + mocks.ensureWorkspace.mockResolvedValue({ + id: "workspace_1", + userId: "knowhere-api-key-dev-user", + namespace: "notebook-workspace_1", + createdAt: new Date("2026-05-10T00:00:00.000Z"), + }) + mocks.findSourceInWorkspace.mockResolvedValue(null) + mocks.fetchDemoChunkPage.mockResolvedValue({ + demoSourceId: "demo-tsla-q4-2025", + canonicalDocumentId: "demo-doc-tsla-q4-2025", + title: "TSLA-Q4-2025-Update.pdf", + mimeType: "application/pdf", + chunks: [ + { + id: "demo-tsla-q4-2025:chunk_1", + chunkId: "chunk_1", + chunkType: "text", + content: "Tesla demo content", + sectionPath: "Summary", + sourceChunkPath: "Summary", + filePath: null, + sortOrder: 0, + metadata: {}, + assetUrl: null, + }, + ], + pagination: { + page: 1, + pageSize: 100, + total: 70, + totalPages: 1, + }, + }) + + const response = await GET( + new NextRequest( + "http://localhost:3001/api/sources/demo-tsla-q4-2025/chunks?page=1&pageSize=100", + ), + { params: Promise.resolve({ sourceId: "demo-tsla-q4-2025" }) }, + ) + + await expect(response.json()).resolves.toMatchObject({ + chunks: [ + { + chunkId: "demo-tsla-q4-2025:chunk_1", + documentId: "demo-doc-tsla-q4-2025", + sourceTitle: "TSLA-Q4-2025-Update.pdf", + }, + ], + pagination: { + page: 1, + pageSize: 100, + total: 70, + }, + }) + expect(response.status).toBe(200) + expect(mocks.fetchDemoChunkPage).toHaveBeenCalledWith({ + demoSourceId: "demo-tsla-q4-2025", + page: 1, + pageSize: 100, + }) + expect(mocks.findSourceInWorkspace).not.toHaveBeenCalled() + expect(mocks.ensureApiKeyForWorkspace).not.toHaveBeenCalled() + expect(mocks.makeKnowhereClient).not.toHaveBeenCalled() + expect(mocks.getSourceParseAssetUrls).not.toHaveBeenCalled() + }) + + it("serves demo chunks for authenticated materialized demo sources", async () => { + mocks.getCurrentUser.mockResolvedValue({ + id: "user_1", + email: null, + name: null, + }) + mocks.ensureWorkspace.mockResolvedValue({ + id: "workspace_1", + userId: "user_1", + namespace: "notebook-workspace_1", + createdAt: new Date("2026-05-10T00:00:00.000Z"), + }) + mocks.findSourceInWorkspace.mockResolvedValue({ + id: "00000000-0000-0000-0000-000000000001", + workspaceId: "workspace_1", + title: "TSLA-Q4-2025-Update.pdf", + mimeType: "application/pdf", + sizeBytes: 1024, + status: "ready", + failureReason: null, + knowhereJobId: null, + knowhereDocumentId: "copied-doc-tsla-q4-2025", + stagedBlobPathname: null, + stagedBlobUrl: null, + originalBlobPathname: null, + originalBlobUrl: "/api/demo-sources/demo-tsla-q4-2025/original", + demoKey: "demo-tsla-q4-2025", + createdAt: new Date("2026-05-10T00:00:00.000Z"), + updatedAt: new Date("2026-05-10T00:00:00.000Z"), + deletedAt: null, + }) + mocks.fetchDemoChunkPage.mockResolvedValue({ + demoSourceId: "demo-tsla-q4-2025", + canonicalDocumentId: "demo-doc-tsla-q4-2025", + title: "TSLA-Q4-2025-Update.pdf", + mimeType: "application/pdf", + chunks: [ + { + id: "demo-tsla-q4-2025:chunk_1", + chunkId: "chunk_1", + chunkType: "text", + content: "Tesla demo content", + sectionPath: "Summary", + sourceChunkPath: "Summary", + filePath: null, + sortOrder: 0, + metadata: {}, + assetUrl: null, + }, + ], + pagination: { + page: 1, + pageSize: 100, + total: 70, + totalPages: 1, + }, + }) + + const response = await GET( + new NextRequest( + "http://localhost:3001/api/sources/00000000-0000-0000-0000-000000000001/chunks?page=1&pageSize=100", + ), + { params: Promise.resolve({ sourceId: "00000000-0000-0000-0000-000000000001" }) }, + ) + + await expect(response.json()).resolves.toMatchObject({ + chunks: [ + { + chunkId: "demo-tsla-q4-2025:chunk_1", + documentId: "copied-doc-tsla-q4-2025", + sourceTitle: "TSLA-Q4-2025-Update.pdf", + }, + ], + pagination: { + page: 1, + pageSize: 100, + total: 70, + }, + }) + expect(response.status).toBe(200) + expect(mocks.fetchDemoChunkPage).toHaveBeenCalledWith({ + demoSourceId: "demo-tsla-q4-2025", + page: 1, + pageSize: 100, + }) + expect(mocks.ensureApiKeyForWorkspace).not.toHaveBeenCalled() + expect(mocks.makeKnowhereClient).not.toHaveBeenCalled() + expect(mocks.getSourceParseAssetUrls).not.toHaveBeenCalled() + }) + + it("logs the demo chunk load failure before returning 404", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined) + try { + mocks.getCurrentUser.mockResolvedValue({ + id: "knowhere-api-key-dev-user", + email: null, + name: "Knowhere API Key Development", + }) + mocks.ensureWorkspace.mockResolvedValue({ + id: "workspace_1", + userId: "knowhere-api-key-dev-user", + namespace: "notebook-workspace_1", + createdAt: new Date("2026-05-10T00:00:00.000Z"), + }) + mocks.findSourceInWorkspace.mockResolvedValue(null) + mocks.fetchDemoChunkPage.mockRejectedValue( + new Error("Knowhere demo API failed: status=404"), + ) + + const response = await GET( + new NextRequest( + "http://localhost:3001/api/sources/demo-tsla-q4-2025/chunks?page=1&pageSize=100", + ), + { params: Promise.resolve({ sourceId: "demo-tsla-q4-2025" }) }, + ) + + expect(response.status).toBe(404) + const line = String(warnSpy.mock.calls[0]?.[0] ?? "") + const log = JSON.parse(line) as { + readonly msg?: unknown + readonly sourceId?: unknown + readonly page?: unknown + readonly pageSize?: unknown + readonly shouldLoadAll?: unknown + readonly error?: unknown + } + expect(log).toMatchObject({ + msg: "sources: demo chunk load failed", + sourceId: "demo-tsla-q4-2025", + page: 1, + pageSize: 100, + shouldLoadAll: false, + error: "Knowhere demo API failed: status=404", + }) + } finally { + warnSpy.mockRestore() + } + }) + it("loads authenticated workspace chunks without probing the demo endpoint first", async () => { const knowhereClient = { documents: { @@ -237,7 +449,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { createdAt: new Date("2026-05-10T00:00:00.000Z"), }) mocks.findSourceInWorkspace.mockResolvedValue({ - id: "source_1", + id: "00000000-0000-0000-0000-000000000002", workspaceId: "workspace_1", title: "notes.pdf", mimeType: "application/pdf", @@ -261,9 +473,9 @@ describe("GET /api/sources/[sourceId]/chunks", () => { const response = await GET( new NextRequest( - "http://localhost:3001/api/sources/source_1/chunks?page=1&pageSize=1", + "http://localhost:3001/api/sources/00000000-0000-0000-0000-000000000002/chunks?page=1&pageSize=1", ), - { params: Promise.resolve({ sourceId: "source_1" }) }, + { params: Promise.resolve({ sourceId: "00000000-0000-0000-0000-000000000002" }) }, ) await expect(response.json()).resolves.toMatchObject({ diff --git a/src/app/api/sources/reconcile/route.ts b/src/app/api/sources/reconcile/route.ts new file mode 100644 index 0000000..c28bfa4 --- /dev/null +++ b/src/app/api/sources/reconcile/route.ts @@ -0,0 +1,50 @@ +import { serve } from "@upstash/workflow/nextjs" + +import { reconcileSourcesForWorkspace } from "@/domains/sources/reconcile" +import { makeKnowhereClient } from "@/integrations/knowhere" +import { logger } from "@/lib/logger" + +type ReconcilePayload = { + readonly workspaceId: string + readonly sourceId: string + readonly apiKey: string +} + +const MAX_POLL_ATTEMPTS = 60 +const INITIAL_DELAY_S = 3 +const MAX_DELAY_S = 30 + +export const { POST } = serve(async (context) => { + const { workspaceId, sourceId, apiKey } = context.requestPayload + const workspace = { id: workspaceId } + const client = makeKnowhereClient(apiKey) + let delay = INITIAL_DELAY_S + + for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt++) { + const resolved = await context.run(`poll-${attempt}`, async () => { + const sources = await reconcileSourcesForWorkspace(workspace, client) + const source = sources.find((s) => s.id === sourceId) + if (!source || source.status !== "parsing") { + return { done: true, status: source?.status ?? "gone" } as const + } + return { done: false } as const + }) + + if (resolved.done) { + logger.info("workflow: source resolved", { + sourceId, + status: resolved.status, + attempts: attempt + 1, + }) + return + } + + await context.sleep(`wait-${attempt}`, delay) + delay = Math.min(Math.round(delay * 1.5), MAX_DELAY_S) + } + + logger.error("workflow: exhausted poll attempts", { + sourceId, + maxAttempts: MAX_POLL_ATTEMPTS, + }) +}) diff --git a/src/components/chunks-panel-state.test.ts b/src/components/chunks-panel-state.test.ts index 81c17a8..114ee4c 100644 --- a/src/components/chunks-panel-state.test.ts +++ b/src/components/chunks-panel-state.test.ts @@ -93,36 +93,4 @@ describe("chunksPanelState", () => { ).toBe("Image 12") }) - it("finds chunk search matches across content, summary, and keywords", () => { - const chunks: ParsedChunkView[] = [ - { - chunkId: "chunk_1", - type: "text", - content: "Tesla storage revenue increased while Tesla deployed Megapack.", - summary: "Storage update", - keywords: ["Energy storage"], - sourceTitle: "report.pdf", - }, - { - chunkId: "chunk_2", - type: "image", - content: "", - summary: "Megapack deployment chart", - keywords: ["Tesla Energy"], - sourceTitle: "report.pdf", - }, - { - chunkId: "chunk_3", - type: "text", - content: "Vehicle deliveries only.", - sourceTitle: "report.pdf", - }, - ] - - expect(chunksPanelState.getChunkSearchMatches(chunks, " tesla ")).toEqual([ - { chunkId: "chunk_1", matchCount: 2 }, - { chunkId: "chunk_2", matchCount: 1 }, - ]) - expect(chunksPanelState.getChunkSearchMatches(chunks, "missing")).toEqual([]) - }) }) diff --git a/src/components/chunks-panel-state.ts b/src/components/chunks-panel-state.ts index d1b101b..430951a 100644 --- a/src/components/chunks-panel-state.ts +++ b/src/components/chunks-panel-state.ts @@ -9,19 +9,10 @@ type RenderableReference = { readonly connection: ParsedChunkConnection } -export type ChunkSearchMatch = { - readonly chunkId: string - readonly matchCount: number -} - type ChunksPanelStateModule = { readonly formatChunkSectionPath: ( sectionPath: ParsedChunkView["sectionPath"], ) => string | null - readonly getChunkSearchMatches: ( - chunks: readonly ParsedChunkView[], - query: string, - ) => readonly ChunkSearchMatch[] readonly formatReferenceLabel: (ref: string) => string readonly getChunksWithFocusedFirst: ( chunks: readonly ParsedChunkView[], @@ -31,7 +22,6 @@ type ChunksPanelStateModule = { readonly getRenderableReferences: ( chunk: ParsedChunkView, ) => RenderableReference[] - readonly normalizeChunkSearchQuery: (query: string) => string } function getChunksWithFocusedFirst( @@ -139,58 +129,6 @@ function getRenderableReferences( return nonOverlapping } -function getChunkSearchMatches( - chunks: readonly ParsedChunkView[], - query: string, -): readonly ChunkSearchMatch[] { - const normalizedQuery = normalizeChunkSearchQuery(query) - if (!normalizedQuery) return [] - - return chunks.flatMap((chunk): ChunkSearchMatch[] => { - const matchCount = countChunkSearchMatches(chunk, normalizedQuery) - if (matchCount === 0) return [] - return [{ chunkId: chunk.chunkId, matchCount }] - }) -} - -function normalizeChunkSearchQuery(query: string): string { - return query.trim().replace(/\s+/g, " ").toLocaleLowerCase() -} - -function countChunkSearchMatches( - chunk: ParsedChunkView, - normalizedQuery: string, -): number { - return getChunkSearchText(chunk).reduce( - (total, text) => total + countTextMatches(text, normalizedQuery), - 0, - ) -} - -function getChunkSearchText(chunk: ParsedChunkView): readonly string[] { - return [ - chunk.content, - chunk.summary ?? "", - ...(chunk.keywords ?? []), - ].filter((text) => text.trim().length > 0) -} - -function countTextMatches(text: string, normalizedQuery: string): number { - const normalizedText = text.toLocaleLowerCase() - let count = 0 - let cursor = 0 - - while (cursor < normalizedText.length) { - const matchIndex = normalizedText.indexOf(normalizedQuery, cursor) - if (matchIndex < 0) return count - - count += 1 - cursor = matchIndex + normalizedQuery.length - } - - return count -} - function getReferenceRange( content: string, connection: ParsedChunkConnection, @@ -253,10 +191,8 @@ function capitalize(value: string): string { export const chunksPanelState: ChunksPanelStateModule = { formatChunkSectionPath, - getChunkSearchMatches, formatReferenceLabel, getChunksWithFocusedFirst, getReferenceLabel, getRenderableReferences, - normalizeChunkSearchQuery, } diff --git a/src/components/chunks-panel-workflow.ts b/src/components/chunks-panel-workflow.ts index 7439077..a0e9a99 100644 --- a/src/components/chunks-panel-workflow.ts +++ b/src/components/chunks-panel-workflow.ts @@ -50,6 +50,12 @@ type ChunksPanelWorkflow = { readonly visibleView: ChunksPanelView } +type LocalFocusedChunk = { + readonly chunkId: string + readonly parentRequestId: number + readonly requestId: number +} + const estimatedChunkCardHeight = 220 const virtualListOverscan = 4 const infiniteScrollThreshold = 720 @@ -67,9 +73,8 @@ export function useChunksPanelWorkflow({ }: ChunksPanelWorkflowInput): ChunksPanelWorkflow { const viewportRef = useRef(null) const [activeView, setActiveView] = useState("parsed") - const [localFocusedChunkId, setLocalFocusedChunkId] = useState( - null, - ) + const [localFocusedChunk, setLocalFocusedChunk] = + useState(null) const [originalTargetPage, setOriginalTargetPage] = useState<{ readonly pageNumber: number | null readonly requestId: number @@ -77,7 +82,14 @@ export function useChunksPanelWorkflow({ pageNumber: null, requestId: 0, }) - const activeFocusedChunkId = focusedChunkId ?? localFocusedChunkId + const activeFocusedChunkId: string | null = + localFocusedChunk?.parentRequestId === focusedChunkRequestId + ? localFocusedChunk.chunkId + : focusedChunkId + const activeFocusedChunkRequestId: number = + localFocusedChunk?.parentRequestId === focusedChunkRequestId + ? localFocusedChunk.requestId + : focusedChunkRequestId const hasOriginalFile = selectedSource !== null && selectedSourceFile !== null const visibleView = hasOriginalFile ? activeView : "parsed" const visibleChunks = useMemo( @@ -157,8 +169,17 @@ export function useChunksPanelWorkflow({ ) const requestChunkFocus = useCallback((chunkId: string | null): void => { - setLocalFocusedChunkId(chunkId) - }, []) + if (chunkId === null) { + setLocalFocusedChunk(null) + return + } + + setLocalFocusedChunk((current: LocalFocusedChunk | null) => ({ + chunkId, + parentRequestId: focusedChunkRequestId, + requestId: (current?.requestId ?? 0) + 1, + })) + }, [focusedChunkRequestId]) const handleChunkSelected = useCallback( (chunk: ParsedChunkView): void => { @@ -198,7 +219,7 @@ export function useChunksPanelWorkflow({ } scrollToFocusedChunk() - }, [activeFocusedChunkId, focusedChunkRequestId, scrollToFocusedChunk]) + }, [activeFocusedChunkId, activeFocusedChunkRequestId, scrollToFocusedChunk]) useEffect(() => { if (!hasOriginalFile) setActiveView("parsed") @@ -208,6 +229,10 @@ export function useChunksPanelWorkflow({ if (focusedChunkId) setActiveView("parsed") }, [focusedChunkId, focusedChunkRequestId]) + useEffect(() => { + setLocalFocusedChunk(null) + }, [selectedSource]) + return { activeFocusedChunkId, handleChunkSelected, diff --git a/src/components/chunks-panel.test.ts b/src/components/chunks-panel.test.ts index 29e55b6..281616f 100644 --- a/src/components/chunks-panel.test.ts +++ b/src/components/chunks-panel.test.ts @@ -623,6 +623,56 @@ describe("ChunksPanel", () => { ).toBe("true"); }); + it("lets in-chunk table references override the current citation focus", async () => { + mockVisibleVirtualViewport(); + const user = userEvent.setup(); + + render( + React.createElement(C, { + chunks: [ + { + chunkId: "text_1", + parserChunkId: "parser_text_1", + type: "text", + content: "See [tables/table-1.html] for Roadster details.", + sourceTitle: "manual.pdf", + connections: [ + { + targetParserChunkId: "parser_table_1", + targetChunkId: "table_1", + relation: "embeds", + ref: "[tables/table-1.html]", + position: { start: 4, end: 25 }, + }, + ], + }, + { + chunkId: "table_1", + parserChunkId: "parser_table_1", + type: "table", + content: + "
RoadsterTBD
", + sourceTitle: "manual.pdf", + }, + ], + selectedSource: "manual.pdf", + focusedChunkId: "text_1", + focusedChunkRequestId: 1, + }), + ); + + await user.click(screen.getByRole("button", { name: "Table 1" })); + + await waitFor(() => { + const focusedRow = screen + .getByTestId("chunk-card-shell-table_1") + .closest("[data-index]"); + + expect(focusedRow?.getAttribute("data-index")).toBe("0"); + expect(focusedRow?.getAttribute("data-focused-chunk")).toBe("true"); + }); + }); + it("renders a focused virtual chunk outside the initial range first", async () => { mockVisibleVirtualViewport(); diff --git a/src/components/chunks-panel.tsx b/src/components/chunks-panel.tsx index 85c6714..6ea3684 100644 --- a/src/components/chunks-panel.tsx +++ b/src/components/chunks-panel.tsx @@ -2,32 +2,21 @@ import { type CSSProperties, - type ChangeEvent, type ReactNode, useCallback, useEffect, - useId, - useMemo, useState, } from "react"; import { type VirtualItem } from "@tanstack/react-virtual"; import { - ChevronDown, - ChevronUp, FilePlus2, Layers, - Search, UploadCloud, - X, } from "lucide-react"; import { ScrollArea } from "@/components/ui/scroll-area"; import { SourceOriginalPreview } from "@/components/source-original-preview"; import { SourceUploadDialog } from "@/components/source-upload-dialog"; import { useChunksPanelWorkflow } from "@/components/chunks-panel-workflow"; -import { - chunksPanelState, - type ChunkSearchMatch, -} from "@/components/chunks-panel-state"; import { ParsedChunkCard } from "@/components/parsed-chunk-card"; import { useSourceOriginalPreviewWarmup } from "@/components/source-original-preview-warmup"; import { sourceOriginalPreviewModel } from "@/components/source-original-preview-model"; @@ -71,32 +60,6 @@ export function ChunksPanel({ const [mountedOriginalPreviewKey, setMountedOriginalPreviewKey] = useState< string | null >(null); - const [searchQuery, setSearchQuery] = useState(""); - const [activeSearchMatchIndex, setActiveSearchMatchIndex] = useState(-1); - const normalizedSearchQuery = chunksPanelState.normalizeChunkSearchQuery( - searchQuery, - ); - const searchMatches = useMemo( - () => chunksPanelState.getChunkSearchMatches(chunks, searchQuery), - [chunks, searchQuery], - ); - const totalSearchHitCount = useMemo( - () => - searchMatches.reduce( - (total, searchMatch) => total + searchMatch.matchCount, - 0, - ), - [searchMatches], - ); - const effectiveActiveSearchMatchIndex = getEffectiveSearchMatchIndex( - activeSearchMatchIndex, - searchMatches, - normalizedSearchQuery, - ); - const activeSearchMatch = - effectiveActiveSearchMatchIndex >= 0 - ? (searchMatches[effectiveActiveSearchMatchIndex] ?? null) - : null; const { activeFocusedChunkId, handleChunkSelected: selectChunk, @@ -125,20 +88,6 @@ export function ChunksPanel({ onLoadMore, }); - useEffect(() => { - if (!normalizedSearchQuery) { - requestChunkFocus(null); - return; - } - - if (!activeSearchMatch) { - requestChunkFocus(null); - return; - } - - requestChunkFocus(activeSearchMatch.chunkId); - }, [activeSearchMatch, normalizedSearchQuery, requestChunkFocus]); - useSourceOriginalPreviewWarmup({ sourceTitle: selectedSource, file: selectedSourceFile, @@ -161,31 +110,6 @@ export function ChunksPanel({ rememberOriginalPreview(); selectOriginalView(); }, [rememberOriginalPreview, selectOriginalView]); - const handleSearchQueryChange = useCallback( - (query: string): void => { - const nextMatches = chunksPanelState.getChunkSearchMatches(chunks, query); - setSearchQuery(query); - setActiveSearchMatchIndex(nextMatches.length > 0 ? 0 : -1); - if (visibleView === "original") handleParsedViewSelected(); - }, - [chunks, handleParsedViewSelected, visibleView], - ); - const handleSearchCleared = useCallback((): void => { - setSearchQuery(""); - setActiveSearchMatchIndex(-1); - requestChunkFocus(null); - }, [requestChunkFocus]); - const handlePreviousSearchMatch = useCallback((): void => { - setActiveSearchMatchIndex((currentIndex) => - getRelativeSearchMatchIndex(currentIndex, searchMatches, -1), - ); - }, [searchMatches]); - const handleNextSearchMatch = useCallback((): void => { - setActiveSearchMatchIndex((currentIndex) => - getRelativeSearchMatchIndex(currentIndex, searchMatches, 1), - ); - }, [searchMatches]); - const headerTitle = focusedChunkId ? "Referenced Chunks" : "Parsed Chunks"; const shouldMountOriginalPreview = visibleView === "original" || @@ -230,20 +154,6 @@ export function ChunksPanel({

- = 0 - ? effectiveActiveSearchMatchIndex + 1 - : 0 - } - matchCount={searchMatches.length} - query={searchQuery} - totalHitCount={totalSearchHitCount} - onClear={handleSearchCleared} - onNext={handleNextSearchMatch} - onPrevious={handlePreviousSearchMatch} - onQueryChange={handleSearchQueryChange} - /> {hasOriginalFile ? (
- ) : null} -
- - {resultLabel} - -
- - -
-
- ); -} - -function getSearchResultLabel({ - activeMatchOrdinal, - hasQuery, - matchCount, - totalHitCount, -}: { - readonly activeMatchOrdinal: number; - readonly hasQuery: boolean; - readonly matchCount: number; - readonly totalHitCount: number; -}): string { - if (!hasQuery) return "Search chunks"; - if (matchCount === 0) return "No matches"; - - const hitLabel = totalHitCount === 1 ? "hit" : "hits"; - return `${activeMatchOrdinal}/${matchCount} chunks ยท ${totalHitCount} ${hitLabel}`; -} - -function getRelativeSearchMatchIndex( - currentIndex: number, - matches: readonly ChunkSearchMatch[], - delta: number, -): number { - if (matches.length === 0) return -1; - - const startingIndex = currentIndex >= 0 ? currentIndex : 0; - return (startingIndex + delta + matches.length) % matches.length; -} - -function getEffectiveSearchMatchIndex( - currentIndex: number, - matches: readonly ChunkSearchMatch[], - normalizedSearchQuery: string, -): number { - if (!normalizedSearchQuery || matches.length === 0) return -1; - if (currentIndex < 0) return 0; - return Math.min(currentIndex, matches.length - 1); -} - -const searchNavigationButtonClassName = - "flex size-8 items-center justify-center rounded-md text-muted-foreground transition-colors hover:bg-background hover:text-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-[#8e51ff]/25 disabled:pointer-events-none disabled:opacity-40"; - function EmptySourceUploadState({ onLoginClick, onSourceUploaded, @@ -614,7 +390,6 @@ function VirtualChunkRow({ chunk, focusedChunkId, isOriginalPreviewAvailable, - searchQuery, measureElement, onChunkClick, onReferenceClick, @@ -623,7 +398,6 @@ function VirtualChunkRow({ chunk: ParsedChunkView | undefined; focusedChunkId: string | null; isOriginalPreviewAvailable: boolean; - searchQuery: string; measureElement: (node: HTMLDivElement | null) => void; onChunkClick?: (chunk: ParsedChunkView) => void; onReferenceClick: (chunkId: string) => void; @@ -651,7 +425,6 @@ function VirtualChunkRow({ chunk={chunk} isFocused={chunk.chunkId === focusedChunkId} isOriginalPreviewAvailable={isOriginalPreviewAvailable} - searchQuery={searchQuery} onChunkClick={onChunkClick} onReferenceClick={onReferenceClick} /> diff --git a/src/components/notebook-logo-mark.tsx b/src/components/notebook-logo-mark.tsx index 1b3f58a..79208dc 100644 --- a/src/components/notebook-logo-mark.tsx +++ b/src/components/notebook-logo-mark.tsx @@ -21,6 +21,7 @@ export function NotebookLogoMark({ width, className }: NotebookLogoMarkProps) { className={className} width={width} height={height} + style={{ height: "auto" }} /> ); } diff --git a/src/components/parsed-chunk-card.test.ts b/src/components/parsed-chunk-card.test.ts index 85283f7..350a1fb 100644 --- a/src/components/parsed-chunk-card.test.ts +++ b/src/components/parsed-chunk-card.test.ts @@ -24,7 +24,6 @@ describe("ParsedChunkCard", () => { keywords: ["Supercharging", "AI training capacity"], }, isFocused: true, - searchQuery: "capacity", onReferenceClick: vi.fn(), }), ); @@ -44,9 +43,6 @@ describe("ParsedChunkCard", () => { expect(screen.getByTestId("chunk-card-shell-text_1").className).toContain( "min-w-0", ); - expect( - container.querySelectorAll('mark[data-chunk-search-match="true"]').length, - ).toBeGreaterThan(0); }); it("routes resolved artifact reference clicks to the target chunk", async () => { diff --git a/src/components/parsed-chunk-card.tsx b/src/components/parsed-chunk-card.tsx index 18921fd..a75d41b 100644 --- a/src/components/parsed-chunk-card.tsx +++ b/src/components/parsed-chunk-card.tsx @@ -23,14 +23,12 @@ export function ParsedChunkCard({ chunk, isFocused, isOriginalPreviewAvailable = false, - searchQuery = "", onChunkClick, onReferenceClick, }: { readonly chunk: ParsedChunkView; readonly isFocused: boolean; readonly isOriginalPreviewAvailable?: boolean; - readonly searchQuery?: string; readonly onChunkClick?: (chunk: ParsedChunkView) => void; readonly onReferenceClick: (chunkId: string) => void; }): ReactNode { @@ -41,7 +39,6 @@ export function ParsedChunkCard({ chunk={chunk} isFocused={isFocused} isOriginalPreviewAvailable={isOriginalPreviewAvailable} - searchQuery={searchQuery} onChunkClick={onChunkClick} /> @@ -54,7 +51,6 @@ export function ParsedChunkCard({ chunk={chunk} isFocused={isFocused} isOriginalPreviewAvailable={isOriginalPreviewAvailable} - searchQuery={searchQuery} onChunkClick={onChunkClick} /> @@ -66,7 +62,6 @@ export function ParsedChunkCard({ chunk={chunk} isFocused={isFocused} isOriginalPreviewAvailable={isOriginalPreviewAvailable} - searchQuery={searchQuery} onChunkClick={onChunkClick} onReferenceClick={onReferenceClick} /> @@ -230,10 +225,8 @@ function getOpenOriginalButtonLabel( function ChunkSummaryPanel({ chunk, - searchQuery, }: { readonly chunk: ParsedChunkView; - readonly searchQuery: string; }): ReactNode { if (!chunk.summary) return null; @@ -244,7 +237,7 @@ function ChunkSummaryPanel({ > } label="Summary" />

- {renderHighlightedSearchText(chunk.summary, searchQuery)} + {chunk.summary}

); @@ -270,10 +263,8 @@ function ChunkContentPanel({ function ChunkKeywords({ chunk, - searchQuery, }: { readonly chunk: ParsedChunkView; - readonly searchQuery: string; }): ReactNode { if (!chunk.keywords || chunk.keywords.length === 0) return null; @@ -295,7 +286,7 @@ function ChunkKeywords({ variant="secondary" className={keywordBadgeClassName} > - {renderHighlightedSearchText(keyword, searchQuery)} + {keyword} ))} @@ -331,14 +322,12 @@ function TextChunkCard({ chunk, isFocused, isOriginalPreviewAvailable, - searchQuery, onChunkClick, onReferenceClick, }: { readonly chunk: ParsedChunkView; readonly isFocused: boolean; readonly isOriginalPreviewAvailable: boolean; - readonly searchQuery: string; readonly onChunkClick?: (chunk: ParsedChunkView) => void; readonly onReferenceClick: (chunkId: string) => void; }): ReactNode { @@ -349,13 +338,13 @@ function TextChunkCard({ isOriginalPreviewAvailable={isOriginalPreviewAvailable} onChunkClick={onChunkClick} > - + 
-          {renderTextChunkContent(chunk, searchQuery, onReferenceClick)}
+          {renderTextChunkContent(chunk, onReferenceClick)}
- + ); } @@ -364,13 +353,11 @@ function ImageChunkCard({ chunk, isFocused, isOriginalPreviewAvailable, - searchQuery, onChunkClick, }: { readonly chunk: ParsedChunkView; readonly isFocused: boolean; readonly isOriginalPreviewAvailable: boolean; - readonly searchQuery: string; readonly onChunkClick?: (chunk: ParsedChunkView) => void; }): ReactNode { return ( @@ -380,7 +367,7 @@ function ImageChunkCard({ isOriginalPreviewAvailable={isOriginalPreviewAvailable} onChunkClick={onChunkClick} > - + {chunk.assetUrl ? (
@@ -400,31 +387,30 @@ function ImageChunkCard({

{chunk.summary - ? renderHighlightedSearchText(chunk.summary, searchQuery) + ? chunk.summary : "Image content is not available in this view."}

)} - + ); } function renderTextChunkContent( chunk: ParsedChunkView, - searchQuery: string, onReferenceClick: (chunkId: string) => void, ): ReactNode { const parts = parsedChunkCardModel.getTextContentParts(chunk); if (parts.length === 1 && parts[0]?.type === "text") { - return renderHighlightedSearchText(parts[0].text, searchQuery); + return parts[0].text; } return parts.map((part) => { if (part.type === "text") { - return renderHighlightedSearchText(part.text, searchQuery); + return part.text; } return ( @@ -464,13 +450,11 @@ function TableChunkCard({ chunk, isFocused, isOriginalPreviewAvailable, - searchQuery, onChunkClick, }: { readonly chunk: ParsedChunkView; readonly isFocused: boolean; readonly isOriginalPreviewAvailable: boolean; - readonly searchQuery: string; readonly onChunkClick?: (chunk: ParsedChunkView) => void; }): ReactNode { const safeHtml = useMemo( @@ -485,7 +469,7 @@ function TableChunkCard({ isOriginalPreviewAvailable={isOriginalPreviewAvailable} onChunkClick={onChunkClick} > - + {safeHtml ? (

{chunk.summary - ? renderHighlightedSearchText(chunk.summary, searchQuery) + ? chunk.summary : "Table content is not available in this view."}

)} - + ); } -function renderHighlightedSearchText( - text: string, - searchQuery: string, -): ReactNode { - const normalizedQuery = searchQuery.trim().toLocaleLowerCase(); - if (!normalizedQuery) return text; - - const normalizedText = text.toLocaleLowerCase(); - const parts: ReactNode[] = []; - let cursor = 0; - let matchIndex = normalizedText.indexOf(normalizedQuery, cursor); - - while (matchIndex >= 0) { - if (matchIndex > cursor) { - parts.push(text.slice(cursor, matchIndex)); - } - - const matchEnd = matchIndex + normalizedQuery.length; - parts.push( - - {text.slice(matchIndex, matchEnd)} - , - ); - cursor = matchEnd; - matchIndex = normalizedText.indexOf(normalizedQuery, cursor); - } - - if (cursor < text.length) { - parts.push(text.slice(cursor)); - } - - return parts.length === 1 ? parts[0] : parts; -} - function renderChunkIcon(type: ParsedChunkView["type"]): ReactNode { if (type === "image") return ; if (type === "table") return ; diff --git a/src/components/source-original-preview.test.ts b/src/components/source-original-preview.test.ts index c673793..0ceb1e9 100644 --- a/src/components/source-original-preview.test.ts +++ b/src/components/source-original-preview.test.ts @@ -566,6 +566,40 @@ describe("SourceOriginalPreview", () => { expect(screen.queryByText(/
/)).toBeNull(); }); + it("keeps Markdown content readable inside the responsive original preview shell", async () => { + vi.stubGlobal( + "fetch", + vi.fn(() => + Promise.resolve( + new Response("# Scan report\n\nThe scan found placeholder keys.", { + status: 200, + }), + ), + ), + ); + + render( + React.createElement(SourceOriginalPreview, { + sourceTitle: "scan-report.md", + file: { + url: "https://store.public.blob.vercel-storage.com/source-uploads/upload_1/scan-report.md", + mimeType: "text/markdown", + }, + }), + ); + + await waitFor(() => { + expect(screen.getByText("Scan report")).toBeTruthy(); + }); + + const previewShell = screen.getByTestId("source-original-preview"); + expect(previewShell.className).toContain("w-[90%]"); + expect(previewShell.className).toContain("max-w-[1600px]"); + + const markdownPreview = document.querySelector(".original-markdown-preview"); + expect(markdownPreview?.parentElement?.className).toContain("max-w-4xl"); + }); + it("hides the download action for non-downloadable demo originals", () => { render( React.createElement(SourceOriginalPreview, { diff --git a/src/components/source-original-preview.tsx b/src/components/source-original-preview.tsx index a105abe..03c69a1 100644 --- a/src/components/source-original-preview.tsx +++ b/src/components/source-original-preview.tsx @@ -49,7 +49,7 @@ export function SourceOriginalPreview({
@@ -84,6 +84,10 @@ export function SourceOriginalPreview({ ); } +function getPreviewShellClassName(): string { + return "mx-auto flex w-[90%] min-w-0 max-w-[1600px] flex-col gap-3 p-3 sm:p-6"; +} + function renderPreview( kind: PreviewKind, sourceTitle: string, @@ -119,9 +123,9 @@ function renderPreview( /> ); case "markdown": - return ; + return renderReadingPreview(file, "markdown"); case "text": - return ; + return renderReadingPreview(file, "text"); case "docx": return ; case "unsupported": @@ -129,6 +133,17 @@ function renderPreview( } } +function renderReadingPreview( + file: SourceOriginalFileView, + variant: "markdown" | "text", +): ReactNode { + return ( +
+ +
+ ); +} + function UnsupportedPreview(): ReactNode { return (
diff --git a/src/components/source-panel-state.test.ts b/src/components/source-panel-state.test.ts index 5bd7eb9..e7d83b8 100644 --- a/src/components/source-panel-state.test.ts +++ b/src/components/source-panel-state.test.ts @@ -23,18 +23,16 @@ describe("sourcePanelState", () => { expect(state.archivingSourceIdSet.has("source_1")).toBe(true); }); - it("selects or clears the current Source from a row click", () => { + it("selects a Source from a row click without clearing the current selection", () => { expect( sourcePanelState.getNextSelectedSourceId({ sourceId: "source_1", - selectedSourceId: null, }), ).toBe("source_1"); expect( sourcePanelState.getNextSelectedSourceId({ sourceId: "source_1", - selectedSourceId: "source_1", }), - ).toBeNull(); + ).toBe("source_1"); }); }); diff --git a/src/components/source-panel-state.ts b/src/components/source-panel-state.ts index 2f88600..07628a8 100644 --- a/src/components/source-panel-state.ts +++ b/src/components/source-panel-state.ts @@ -13,7 +13,6 @@ type ArchiveConfirmationState = { } type NextSelectedSourceInput = { - readonly selectedSourceId: string | null readonly sourceId: string } @@ -48,11 +47,8 @@ function getArchiveConfirmationState({ } } -function getNextSelectedSourceId({ - selectedSourceId, - sourceId, -}: NextSelectedSourceInput): string | null { - return sourceId === selectedSourceId ? null : sourceId +function getNextSelectedSourceId({ sourceId }: NextSelectedSourceInput): string | null { + return sourceId } function shouldCloseArchiveConfirmation( diff --git a/src/components/sources-panel.tsx b/src/components/sources-panel.tsx index 0f96003..e947c49 100644 --- a/src/components/sources-panel.tsx +++ b/src/components/sources-panel.tsx @@ -137,7 +137,6 @@ export function SourcesPanel({ onSelectSource?.( sourcePanelState.getNextSelectedSourceId({ sourceId: source.id, - selectedSourceId, }), ) } diff --git a/src/components/top-nav.tsx b/src/components/top-nav.tsx index e33c20b..487fcc2 100644 --- a/src/components/top-nav.tsx +++ b/src/components/top-nav.tsx @@ -1,7 +1,7 @@ import { NotebookLogoMark } from "@/components/notebook-logo-mark"; import { Separator } from "@/components/ui/separator"; import { ThemeToggle } from "@/components/theme-toggle"; -import { ExternalLink, LayoutDashboard } from "lucide-react"; +import { ExternalLink } from "lucide-react"; import type { ReactElement } from "react"; export type TopNavProps = { @@ -39,9 +39,8 @@ export function TopNav({ - Dashboard diff --git a/src/components/workspace-chat-workflow.ts b/src/components/workspace-chat-workflow.ts index 63ad9eb..d50747f 100644 --- a/src/components/workspace-chat-workflow.ts +++ b/src/components/workspace-chat-workflow.ts @@ -218,12 +218,25 @@ export function useWorkspaceChatWorkflow({ const demoSourceIds = getMaterializableDemoSourceIds(sources) if (demoSourceIds.length > 0) { setChat((current) => - workspaceChatState.prepareSend(current, "Preparing demo sources..."), + workspaceChatState.prepareSend(current, "Thinking"), ) try { const materializedSources = await workspaceClient.materializeDemoSources({ demoSourceIds }) onSourcesMaterialized?.(demoSourceIds, materializedSources) + if (chat.threadId) { + try { + const fresh = await workspaceClient.fetchChatThread(chat.threadId) + setChat((current) => { + if (current.threadId !== fresh.requestedThreadId) return current + if (!fresh.thread || !Array.isArray(fresh.messages)) + return current + return { ...current, messages: [...fresh.messages] } + }) + } catch { + // stale citations until page reload โ€” materialization succeeded + } + } } catch { setChat((current) => ({ ...current, diff --git a/src/components/workspace-citation-focus.ts b/src/components/workspace-citation-focus.ts index 7ca7a0f..d428bf0 100644 --- a/src/components/workspace-citation-focus.ts +++ b/src/components/workspace-citation-focus.ts @@ -1,10 +1,9 @@ "use client" -import { useCallback, useEffect, useState } from "react" +import { useCallback, useState } from "react" import { workspaceCitationState } from "@/components/workspace-citation-state" import { useWorkspaceSelectedChunks } from "@/components/workspace-selected-chunks" -import { useHashFragment } from "@/lib/use-hash-fragment" import type { ChatCitationView } from "@/domains/chat/types" import type { ParsedChunkView } from "@/domains/chunks/types" import type { SourceView } from "@/domains/sources/types" @@ -58,7 +57,6 @@ export function useWorkspaceCitationFocus({ ) const [prefetchedChunksBySourceId, setPrefetchedChunksBySourceId] = useState(initialPrefetchedChunksBySourceId) - const [hashChunkId, setHashChunkId] = useHashFragment() const { hasMoreSelectedChunks, handleLoadMoreChunks, @@ -78,21 +76,10 @@ export function useWorkspaceCitationFocus({ chunkId, requestId: current.requestId + 1, })) - setHashChunkId(chunkId) }, - [setHashChunkId], + [], ) - useEffect(() => { - if (!hashChunkId) return - - const frameId = window.requestAnimationFrame(() => { - requestChunkFocus(hashChunkId) - }) - - return () => window.cancelAnimationFrame(frameId) - }, [hashChunkId, requestChunkFocus]) - const handleSourceSelected = useCallback( (sourceId: string | null): void => { onSelectSource(sourceId) diff --git a/src/components/workspace-shell.test.ts b/src/components/workspace-shell.test.ts index ea5255a..a4e1909 100644 --- a/src/components/workspace-shell.test.ts +++ b/src/components/workspace-shell.test.ts @@ -159,6 +159,64 @@ describe("WorkspaceShell", () => { ).toBeTruthy(); }); + it("shows the first ready document chunks on workspace load", async () => { + const fetch = vi.fn(async (input) => { + const url = getRequestURL(input); + + if (url.pathname === "/api/sources/source_1/chunks") { + return Response.json({ + chunks: [ + { + chunkId: "source_1:chunk_1", + documentId: "doc_1", + sectionPath: "Overview", + type: "text", + content: "First document chunk content.", + sourceTitle: "first.pdf", + }, + ], + pagination: { + page: Number(url.searchParams.get("page") ?? "1"), + pageSize: 100, + total: 1, + totalPages: 1, + }, + }); + } + + return Response.json({ message: "Unexpected request" }, { status: 404 }); + }); + vi.stubGlobal("fetch", fetch); + + render( + React.createElement(C, { + sources: [ + { + id: "source_1", + title: "first.pdf", + status: "ready", + documentId: "doc_1", + }, + { + id: "source_2", + title: "second.pdf", + status: "ready", + documentId: "doc_2", + }, + ], + }), + ); + + const desktopChunksPanel = within(screen.getByTestId("desktop-chunks-panel")); + await waitFor(() => { + expect( + desktopChunksPanel.getByText("First document chunk content."), + ).toBeTruthy(); + }); + expect(countFetches(fetch, "/api/sources/source_1/chunks")).toBe(1); + expect(countFetches(fetch, "/api/sources/source_2/chunks")).toBe(0); + }); + it("focuses guest citations on desktop using loaded demo chunks", async () => { const fetch = vi.fn(async (input) => { const url = getRequestURL(input); diff --git a/src/components/workspace-source-state.test.ts b/src/components/workspace-source-state.test.ts index ccae9e8..7f1596e 100644 --- a/src/components/workspace-source-state.test.ts +++ b/src/components/workspace-source-state.test.ts @@ -5,6 +5,29 @@ import { workspaceSourceState } from "./workspace-source-state"; import type { SourceView } from "@/domains/sources/types"; describe("workspaceSourceState", () => { + it("selects the first ready Source as the initial Source", () => { + const sources: readonly SourceView[] = [ + { + id: "source_parsing", + title: "pending.pdf", + status: "parsing", + mimeType: "application/pdf", + excludedFromQuery: false, + }, + { + id: "source_ready", + title: "ready.pdf", + status: "ready", + mimeType: "application/pdf", + excludedFromQuery: false, + }, + ]; + + expect(workspaceSourceState.getInitialSelectedSourceId(sources)).toBe( + "source_ready", + ); + }); + it("applies source query exclusions without mutating the source list", () => { const sources: readonly SourceView[] = [ { @@ -40,10 +63,27 @@ describe("workspaceSourceState", () => { expect(sources[1]?.excludedFromQuery).toBe(false); }); - it("clears selected and exclusion state when the selected source is archived", () => { + it("moves selection to the first remaining ready Source when the selected Source is archived", () => { + const sources: readonly SourceView[] = [ + { + id: "source_1", + title: "selected.pdf", + status: "ready", + mimeType: "application/pdf", + excludedFromQuery: false, + }, + { + id: "source_2", + title: "remaining.pdf", + status: "ready", + mimeType: "application/pdf", + excludedFromQuery: false, + }, + ]; const result = workspaceSourceState.archiveSource({ sourceId: "source_1", selectedSourceId: "source_1", + sources, sourceExclusionById: { source_1: true, source_2: false, @@ -51,7 +91,7 @@ describe("workspaceSourceState", () => { }); expect(result).toEqual({ - selectedSourceId: null, + selectedSourceId: "source_2", sourceExclusionById: { source_2: false, }, diff --git a/src/components/workspace-source-state.ts b/src/components/workspace-source-state.ts index 9f24185..d4f82d1 100644 --- a/src/components/workspace-source-state.ts +++ b/src/components/workspace-source-state.ts @@ -5,6 +5,7 @@ type SourceExclusionState = Readonly> type ArchiveSourceInput = { readonly sourceId: string readonly selectedSourceId: string | null + readonly sources: readonly SourceView[] readonly sourceExclusionById: SourceExclusionState } @@ -14,9 +15,15 @@ type ArchiveSourceResult = { } type WorkspaceSourceStateModule = { + readonly getFirstReadySourceId: ( + sources: readonly SourceView[], + ) => string | null readonly getInitialSelectedSourceId: ( sources: readonly SourceView[], - isGuest: boolean, + ) => string | null + readonly getResolvedSelectedSourceId: ( + sources: readonly SourceView[], + selectedSourceId: string | null, ) => string | null readonly applyQueryExclusions: ( sources: readonly SourceView[], @@ -38,13 +45,22 @@ type WorkspaceSourceStateModule = { ) => Record } -function getInitialSelectedSourceId( +function getInitialSelectedSourceId(sources: readonly SourceView[]): string | null { + return getFirstReadySourceId(sources) +} + +function getFirstReadySourceId(sources: readonly SourceView[]): string | null { + return sources.find((source) => source.status === "ready")?.id ?? null +} + +function getResolvedSelectedSourceId( sources: readonly SourceView[], - isGuest: boolean, + selectedSourceId: string | null, ): string | null { - if (!isGuest) return null + const selectedSource = sources.find((source) => source.id === selectedSourceId) + if (selectedSource?.status === "ready") return selectedSource.id - return sources.find((source) => source.status === "ready")?.id ?? null + return getFirstReadySourceId(sources) } function applyQueryExclusions( @@ -66,9 +82,14 @@ function upsertSource( } function archiveSource(input: ArchiveSourceInput): ArchiveSourceResult { + const remainingSources = input.sources.filter( + (source) => source.id !== input.sourceId, + ) return { selectedSourceId: - input.selectedSourceId === input.sourceId ? null : input.selectedSourceId, + input.selectedSourceId === input.sourceId + ? getFirstReadySourceId(remainingSources) + : getResolvedSelectedSourceId(remainingSources, input.selectedSourceId), sourceExclusionById: removeRecordKey( input.sourceExclusionById, input.sourceId, @@ -96,7 +117,9 @@ function removeRecordKey( } export const workspaceSourceState: WorkspaceSourceStateModule = { + getFirstReadySourceId, getInitialSelectedSourceId, + getResolvedSelectedSourceId, applyQueryExclusions, upsertSource, archiveSource, diff --git a/src/components/workspace-source-workflow.test.ts b/src/components/workspace-source-workflow.test.ts index 0960704..563f5f0 100644 --- a/src/components/workspace-source-workflow.test.ts +++ b/src/components/workspace-source-workflow.test.ts @@ -53,7 +53,7 @@ describe("useWorkspaceSourceWorkflow", () => { expect(mocks.archiveSource).toHaveBeenCalledWith("source_1") await waitFor(() => { - expect(result.current.selectedSourceId).toBeNull() + expect(result.current.selectedSourceId).toBe("source_2") }) expect(result.current.sources.map((source) => source.id)).toEqual([ "source_2", diff --git a/src/components/workspace-source-workflow.ts b/src/components/workspace-source-workflow.ts index c8f1205..b31380b 100644 --- a/src/components/workspace-source-workflow.ts +++ b/src/components/workspace-source-workflow.ts @@ -41,7 +41,6 @@ export function useWorkspaceSourceWorkflow({ const initialSourceRows = useMemo(() => [...initialSources], [initialSources]) const initialSelectedSourceId = workspaceSourceState.getInitialSelectedSourceId( initialSourceRows, - isGuest, ) const [selectedSourceId, setSelectedSourceId] = useState( initialSelectedSourceId, @@ -68,6 +67,11 @@ export function useWorkspaceSourceWorkflow({ sourceRows, sourceExclusionById, ) + const resolvedSelectedSourceId = + workspaceSourceState.getResolvedSelectedSourceId( + sourceRows, + selectedSourceId, + ) const sourceTitlesByDocumentId = useMemo>>( () => Object.fromEntries( @@ -142,6 +146,7 @@ export function useWorkspaceSourceWorkflow({ workspaceSourceState.archiveSource({ sourceId, selectedSourceId: current, + sources: sourceRows, sourceExclusionById, }).selectedSourceId, ) @@ -149,6 +154,7 @@ export function useWorkspaceSourceWorkflow({ workspaceSourceState.archiveSource({ sourceId, selectedSourceId, + sources: sourceRows, sourceExclusionById: current, }).sourceExclusionById, ) @@ -169,7 +175,7 @@ export function useWorkspaceSourceWorkflow({ handleSourceUploaded, handleToggleIncluded, readySourceCount, - selectedSourceId, + selectedSourceId: resolvedSelectedSourceId, setSelectedSourceId, sourceTitlesByDocumentId, sources, diff --git a/src/domains/chat/chat-citation-persistence.ts b/src/domains/chat/chat-citation-persistence.ts index 498bcc1..a705362 100644 --- a/src/domains/chat/chat-citation-persistence.ts +++ b/src/domains/chat/chat-citation-persistence.ts @@ -13,7 +13,7 @@ type ChatCitationPersistence = { ) => CitationView[] | null readonly replaceDemoCitationDocumentId: ( citations: readonly ChatCitationView[] | undefined, - documentId: string, + documentIdMap: ReadonlyMap, ) => ChatCitationView[] | undefined } @@ -29,17 +29,24 @@ function normalizeCitations( function replaceDemoCitationDocumentId( citations: readonly ChatCitationView[] | undefined, - documentId: string, + documentIdMap: ReadonlyMap, ): ChatCitationView[] | undefined { if (!citations) return undefined - return citations.map((citation) => ({ - ...citation, - source: { - ...citation.source, - documentId, - }, - })) + return citations.map((citation) => { + const newId = citation.source.documentId + ? documentIdMap.get(citation.source.documentId) + : undefined + if (!newId) return citation + + return { + ...citation, + source: { + ...citation.source, + documentId: newId, + }, + } + }) } function toCitationView( diff --git a/src/domains/chat/chat-message-repository.ts b/src/domains/chat/chat-message-repository.ts index 66efa1c..4cc04f1 100644 --- a/src/domains/chat/chat-message-repository.ts +++ b/src/domains/chat/chat-message-repository.ts @@ -32,6 +32,10 @@ type ChatMessageRepository = { workspaceId: string, input: AppendChatMessageInput, ) => Effect.Effect + readonly updateMessageCitationsEffect: ( + messageId: string, + citations: CitationView[] | null, + ) => Effect.Effect } const listMessagesForThreadEffect: ChatMessageRepository["listMessagesForThreadEffect"] = @@ -92,7 +96,22 @@ const appendMessageToThreadEffect: ChatMessageRepository["appendMessageToThreadE ) }) +const updateMessageCitationsEffect: ChatMessageRepository["updateMessageCitationsEffect"] = + (messageId: string, citations: CitationView[] | null) => + Effect.gen(function* () { + const db = yield* DbClient + const [updated] = yield* Effect.promise(() => + db + .update(chatMessages) + .set({ citations }) + .where(eq(chatMessages.id, messageId)) + .returning(), + ) + return updated ?? null + }) + export const chatMessageRepository: ChatMessageRepository = { listMessagesForThreadEffect, appendMessageToThreadEffect, + updateMessageCitationsEffect, } diff --git a/src/domains/chat/chat-thread-repository.ts b/src/domains/chat/chat-thread-repository.ts index 965aee9..d7cc884 100644 --- a/src/domains/chat/chat-thread-repository.ts +++ b/src/domains/chat/chat-thread-repository.ts @@ -3,8 +3,32 @@ import "server-only" import { and, desc, eq, isNull, sql } from "drizzle-orm" import { Effect } from "effect" +import { chatCitationPersistence } from "./chat-citation-persistence" import { DbClient } from "@/infrastructure/db" -import { chatThreads, type ChatThread } from "@/infrastructure/db/schema" +import { + chatMessages, + chatThreads, + type ChatMessage, + type ChatThread, +} from "@/infrastructure/db/schema" +import type { ChatCitationView } from "./types" + +type SeedDemoChatMessage = { + readonly role: "user" | "assistant" + readonly content: string + readonly citations?: readonly ChatCitationView[] | null +} + +type SeedDemoChatThreadInput = { + readonly demoKey: string + readonly title: string + readonly messages: readonly SeedDemoChatMessage[] +} + +type SeedDemoChatThreadResult = { + readonly thread: ChatThread + readonly messages: ChatMessage[] +} type ChatThreadRepository = { readonly findThreadInWorkspaceEffect: ( @@ -20,10 +44,18 @@ type ChatThreadRepository = { readonly ensureDefaultThreadEffect: ( workspaceId: string, ) => Effect.Effect + readonly ensureDemoThreadEffect: ( + workspaceId: string, + input: SeedDemoChatThreadInput, + ) => Effect.Effect readonly softDeleteThreadEffect: ( workspaceId: string, threadId: string, ) => Effect.Effect + readonly findThreadByDemoKeyEffect: ( + workspaceId: string, + demoKey: string, + ) => Effect.Effect } const chatThreadListLimit = 50 @@ -116,6 +148,88 @@ const ensureDefaultThreadEffect: ChatThreadRepository["ensureDefaultThreadEffect return thread }) +const ensureDemoThreadEffect: ChatThreadRepository["ensureDemoThreadEffect"] = + (workspaceId: string, input: SeedDemoChatThreadInput) => + Effect.gen(function* () { + if (input.messages.length === 0) return null + + const db = yield* DbClient + return yield* Effect.promise(() => + db.transaction(async (tx) => { + const insertDemoMessages = async ( + threadId: string, + ): Promise => { + const createdAtMs = Date.now() + return await tx + .insert(chatMessages) + .values( + input.messages.map((message, index) => ({ + threadId, + role: message.role, + content: message.content, + citations: chatCitationPersistence.normalizeCitations( + message.citations, + ), + createdAt: new Date(createdAtMs + index), + })), + ) + .returning() + } + + const existing = ( + await tx + .select() + .from(chatThreads) + .where( + and( + eq(chatThreads.workspaceId, workspaceId), + eq(chatThreads.demoKey, input.demoKey), + ), + ) + .limit(1) + )[0] + + if (existing) { + if (existing.deletedAt !== null) return null + + const existingMessages = await tx + .select() + .from(chatMessages) + .where(eq(chatMessages.threadId, existing.id)) + .orderBy(chatMessages.createdAt) + if (existingMessages.length > 0) { + return { + thread: existing, + messages: existingMessages, + } + } + + const messages = await insertDemoMessages(existing.id) + return { + thread: existing, + messages, + } + } + + const [thread] = await tx + .insert(chatThreads) + .values({ + workspaceId, + demoKey: input.demoKey, + title: input.title, + }) + .returning() + + if (!thread) { + throw new Error("ensureDemoChatThread: insert did not return a row.") + } + + const messages = await insertDemoMessages(thread.id) + return { thread, messages } + }), + ) + }) + const softDeleteThreadEffect: ChatThreadRepository["softDeleteThreadEffect"] = ( workspaceId: string, threadId: string, @@ -139,10 +253,32 @@ const softDeleteThreadEffect: ChatThreadRepository["softDeleteThreadEffect"] = ( return result.length > 0 }) +const findThreadByDemoKeyEffect: ChatThreadRepository["findThreadByDemoKeyEffect"] = + (workspaceId: string, demoKey: string) => + Effect.gen(function* () { + const db = yield* DbClient + const row = yield* Effect.promise(() => + db + .select() + .from(chatThreads) + .where( + and( + eq(chatThreads.workspaceId, workspaceId), + eq(chatThreads.demoKey, demoKey), + isNull(chatThreads.deletedAt), + ), + ) + .limit(1), + ) + return row[0] ?? null + }) + export const chatThreadRepository: ChatThreadRepository = { findThreadInWorkspaceEffect, listThreadsForWorkspaceEffect, createThreadEffect, ensureDefaultThreadEffect, + ensureDemoThreadEffect, softDeleteThreadEffect, + findThreadByDemoKeyEffect, } diff --git a/src/domains/chat/chat-turn-persistence.ts b/src/domains/chat/chat-turn-persistence.ts index c2852dc..fbb759c 100644 --- a/src/domains/chat/chat-turn-persistence.ts +++ b/src/domains/chat/chat-turn-persistence.ts @@ -44,13 +44,19 @@ function createRepository( adapter: ChatThreadPersistenceAdapter = chatThreadService, ): ChatRepository { return { - ensureDefaultChatThread: adapter.ensureDefault, - findChatThreadInWorkspace: adapter.findInWorkspace, - listMessagesForThread: async (workspaceId: string, threadId: string) => { + ensureDefaultChatThread: (workspaceId) => + adapter.ensureDefault(workspaceId), + + findChatThreadInWorkspace: (workspaceId, threadId) => + adapter.findInWorkspace(workspaceId, threadId), + + listMessagesForThread: async (workspaceId, threadId) => { const messages = await adapter.listMessages(workspaceId, threadId) return messages ? [...messages] : null }, - appendMessageToThread: adapter.appendMessage, + + appendMessageToThread: (workspaceId, input) => + adapter.appendMessage(workspaceId, input), } } diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index 90627e7..835fabf 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -340,6 +340,23 @@ describe("buildGroundedPrompt", () => { expect(prompt).toContain("requirements.txt"); expect(prompt).toContain("don't answer the question, say so directly"); }); + + it("asks the model to answer naturally and directly", () => { + const prompt = buildGroundedPrompt({ + question: "How about the TBD?", + results: [ + makeRetrievalResult({ + content: "Roadster location: TBD. Status: Design development.", + }), + ], + }); + + expect(prompt).toContain("Answer in a natural, friendly, and direct tone."); + expect(prompt).toContain("Start with the answer first."); + expect(prompt).toContain("Avoid meta phrases like \"Based on the sources\""); + expect(prompt).toContain("Keep answers concise by default"); + expect(prompt).toContain("I don't see more detail in these sources"); + }); }); describe("buildRetrievalQueryPrompt", () => { diff --git a/src/domains/chat/prompt.ts b/src/domains/chat/prompt.ts index 0620306..ab715e4 100644 --- a/src/domains/chat/prompt.ts +++ b/src/domains/chat/prompt.ts @@ -144,6 +144,11 @@ export function buildGroundedPrompt(input: BuildGroundedPromptInput): string { "You are an assistant that answers questions from provided source excerpts.", "Your answer must be grounded only in the sources below. If they don't answer the question, say so directly.", "Use the recent conversation only to resolve references like \"this document\"; do not use it as factual evidence.", + "Answer in a natural, friendly, and direct tone.", + "Start with the answer first. Avoid meta phrases like \"Based on the sources\" or \"Based on the source excerpts\" unless the user asks how you know.", + "Use plain language. Prefer \"I don't see more detail in these sources\" over formal wording like \"the sources do not specify\".", + "Keep answers concise by default: 1-3 short paragraphs unless the user asks for detail.", + "Do not over-explain uncertainty. State what is known, then briefly state what is not shown in the sources.", "CITATION FORMAT: After each sourced statement include a brief citation label like [Source N: what the source says]. Use only the provided source numbers.", "", `Question: ${input.question}`, diff --git a/src/domains/chat/repository.ts b/src/domains/chat/repository.ts index 1e49080..30ed1a5 100644 --- a/src/domains/chat/repository.ts +++ b/src/domains/chat/repository.ts @@ -8,6 +8,7 @@ type ChatRepository = { readonly listThreadsForWorkspaceEffect: typeof chatThreadRepository.listThreadsForWorkspaceEffect readonly createThreadEffect: typeof chatThreadRepository.createThreadEffect readonly ensureDefaultThreadEffect: typeof chatThreadRepository.ensureDefaultThreadEffect + readonly ensureDemoThreadEffect: typeof chatThreadRepository.ensureDemoThreadEffect readonly listMessagesForThreadEffect: typeof chatMessageRepository.listMessagesForThreadEffect readonly softDeleteThreadEffect: typeof chatThreadRepository.softDeleteThreadEffect readonly appendMessageToThreadEffect: typeof chatMessageRepository.appendMessageToThreadEffect @@ -18,6 +19,7 @@ export const chatRepository: ChatRepository = { listThreadsForWorkspaceEffect: chatThreadRepository.listThreadsForWorkspaceEffect, createThreadEffect: chatThreadRepository.createThreadEffect, ensureDefaultThreadEffect: chatThreadRepository.ensureDefaultThreadEffect, + ensureDemoThreadEffect: chatThreadRepository.ensureDemoThreadEffect, listMessagesForThreadEffect: chatMessageRepository.listMessagesForThreadEffect, softDeleteThreadEffect: chatThreadRepository.softDeleteThreadEffect, appendMessageToThreadEffect: chatMessageRepository.appendMessageToThreadEffect, diff --git a/src/domains/chat/route-answer.ts b/src/domains/chat/route-answer.ts index 896c026..a7599d8 100644 --- a/src/domains/chat/route-answer.ts +++ b/src/domains/chat/route-answer.ts @@ -1,4 +1,4 @@ -import { Either } from "effect" +import { Effect, Either } from "effect" import { generateContextualRetrievalQuery, @@ -30,42 +30,62 @@ type ChatAnswerRouteService = { ) => Promise> } -async function answerChat( - input: AnswerChatInput, -): Promise> { - const body = parseChatRequestBody(input.body) - if (!body.ok) { - return routeResult.error(body.status, body.message) - } +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- - const { workspace, client } = - await notebookRequestContext.getAuthenticatedWithClient() - const sources = await reconcileSourcesForWorkspace(workspace, client) +const answerChatEffect = (input: AnswerChatInput) => + Effect.gen(function* () { + const body = parseChatRequestBody(input.body) + if (!body.ok) { + return routeResult.error(body.status, body.message) + } - try { - const result = await handleChatTurn({ - workspace, - sources, - question: body.value.question, - threadId: body.value.threadId, - excludedSourceIds: body.value.excludedSourceIds, - retrieval: client.retrieval, - generateRetrievalQuery: generateContextualRetrievalQuery, - generateAnswer: generateGroundedAnswer, - repository: chatTurnPersistence.createRepository(), - }) + const { workspace, client } = yield* Effect.tryPromise(() => + notebookRequestContext.getAuthenticatedWithClient(), + ) + const sources = yield* Effect.tryPromise(() => + reconcileSourcesForWorkspace(workspace, client), + ) + + const result = yield* Effect.tryPromise(() => + handleChatTurn({ + workspace, + sources, + question: body.value.question, + threadId: body.value.threadId, + excludedSourceIds: body.value.excludedSourceIds, + retrieval: client.retrieval, + generateRetrievalQuery: generateContextualRetrievalQuery, + generateAnswer: generateGroundedAnswer, + repository: chatTurnPersistence.createRepository(), + }), + ).pipe( + Effect.catchAll(() => + Effect.succeed( + Either.left({ + status: 401, + message: "Your session may have expired. Please refresh the page.", + }), + ), + ), + ) return Either.match(result, { onLeft: (error): RouteResponse => routeResult.error(error.status, error.message), onRight: (value): RouteResponse => routeResult.ok(value), }) - } catch { - return routeResult.error( - 401, - "Your session may have expired. Please refresh the page.", - ) - } + }) + +// --------------------------------------------------------------------------- +// Async wrapper (backward-compatible) +// --------------------------------------------------------------------------- + +async function answerChat( + input: AnswerChatInput, +): Promise> { + return Effect.runPromise(answerChatEffect(input)) } export const chatAnswerRouteService: ChatAnswerRouteService = { diff --git a/src/domains/chat/route-threads.ts b/src/domains/chat/route-threads.ts index 796996d..3a851a6 100644 --- a/src/domains/chat/route-threads.ts +++ b/src/domains/chat/route-threads.ts @@ -1,3 +1,5 @@ +import { Effect } from "effect" + import { chatThreadService } from "@/domains/chat/thread-service" import { toChatMessageView, toChatThreadView } from "@/domains/chat/view" import { notebookRequestContext } from "@/domains/workspace/request-context" @@ -48,67 +50,102 @@ type ChatThreadRouteService = { readonly listThreads: () => Promise> } -async function listThreads(): Promise> { - const { workspace } = await notebookRequestContext.getAuthenticated() - const threads = await chatThreadService.listForWorkspace(workspace.id) +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const listThreadsEffect = Effect.gen(function* () { + const { workspace } = yield* Effect.tryPromise(() => + notebookRequestContext.getAuthenticated(), + ) + const threads = yield* Effect.tryPromise(() => + chatThreadService.listForWorkspace(workspace.id), + ) return routeResult.ok({ threads: threads.map(toChatThreadView), }) -} +}) -async function createThread(): Promise> { - const { workspace } = await notebookRequestContext.getAuthenticated() - const thread = await chatThreadService.create(workspace.id) +const createThreadEffect = Effect.gen(function* () { + const { workspace } = yield* Effect.tryPromise(() => + notebookRequestContext.getAuthenticated(), + ) + const thread = yield* Effect.tryPromise(() => + chatThreadService.create(workspace.id), + ) return routeResult.ok({ thread: toChatThreadView(thread), - messages: [], + messages: [] as unknown as [], }) +}) + +const getThreadEffect = (input: ThreadInput) => + Effect.gen(function* () { + const { workspace } = yield* Effect.tryPromise(() => + notebookRequestContext.getAuthenticated(), + ) + const thread = yield* Effect.tryPromise(() => + chatThreadService.findInWorkspace(workspace.id, input.threadId), + ) + + if (!thread) { + return routeResult.error(404, "Chat thread not found.") + } + + const messages = yield* Effect.tryPromise(() => + chatThreadService.listMessages(workspace.id, input.threadId), + ) + if (!messages) { + return routeResult.error(404, "Chat thread not found.") + } + + return routeResult.ok({ + thread: toChatThreadView(thread), + messages: messages.map((message): ChatMessageView => + toChatMessageView(message), + ), + }) + }) + +const archiveThreadEffect = (input: ArchiveThreadInput) => + Effect.gen(function* () { + const { workspace } = yield* Effect.tryPromise(() => + notebookRequestContext.getAuthenticated(), + ) + const archived = yield* Effect.tryPromise(() => + chatThreadService.softDelete(workspace.id, input.threadId), + ) + if (!archived) { + return routeResult.error(404, "Chat thread not found.") + } + + return routeResult.ok({ id: input.threadId, archived: true as const }) + }) + +// --------------------------------------------------------------------------- +// Async wrappers (backward-compatible) +// --------------------------------------------------------------------------- + +async function listThreads(): Promise> { + return Effect.runPromise(listThreadsEffect) +} + +async function createThread(): Promise> { + return Effect.runPromise(createThreadEffect) } async function getThread( input: ThreadInput, ): Promise> { - const { workspace } = await notebookRequestContext.getAuthenticated() - const thread = await chatThreadService.findInWorkspace( - workspace.id, - input.threadId, - ) - - if (!thread) { - return routeResult.error(404, "Chat thread not found.") - } - - const messages = await chatThreadService.listMessages( - workspace.id, - input.threadId, - ) - if (!messages) { - return routeResult.error(404, "Chat thread not found.") - } - - return routeResult.ok({ - thread: toChatThreadView(thread), - messages: messages.map((message): ChatMessageView => - toChatMessageView(message), - ), - }) + return Effect.runPromise(getThreadEffect(input)) } async function archiveThread( input: ArchiveThreadInput, ): Promise> { - const { workspace } = await notebookRequestContext.getAuthenticated() - const archived = await chatThreadService.softDelete( - workspace.id, - input.threadId, - ) - if (!archived) { - return routeResult.error(404, "Chat thread not found.") - } - - return routeResult.ok({ id: input.threadId, archived: true }) + return Effect.runPromise(archiveThreadEffect(input)) } export const chatThreadRouteService: ChatThreadRouteService = { diff --git a/src/domains/chat/thread-service.ts b/src/domains/chat/thread-service.ts index b5f3275..e22085f 100644 --- a/src/domains/chat/thread-service.ts +++ b/src/domains/chat/thread-service.ts @@ -1,8 +1,10 @@ import "server-only" import { databaseRuntime } from "@/domains/workspace/database-runtime" +import { demoView } from "@/domains/demo/view" import { chatRepository } from "./repository" import type { ChatMessage, ChatThread } from "@/infrastructure/db/schema" +import type { DemoCatalog } from "@/integrations/knowhere-demo" import type { ChatCitationView, CitationView, @@ -18,6 +20,11 @@ type AppendMessageInput = { | null } +type DemoChatThreadSeed = { + readonly thread: ChatThread + readonly messages: ChatMessage[] +} + type ChatThreadService = { readonly findInWorkspace: ( workspaceId: string, @@ -26,6 +33,10 @@ type ChatThreadService = { readonly listForWorkspace: (workspaceId: string) => Promise readonly create: (workspaceId: string) => Promise readonly ensureDefault: (workspaceId: string) => Promise + readonly ensureDemo: ( + workspaceId: string, + catalog: DemoCatalog, + ) => Promise readonly listMessages: ( workspaceId: string, threadId: string, @@ -40,6 +51,8 @@ type ChatThreadService = { ) => Promise } +const seededDemoChatKey = "knowhere-demo-chat" + const findInWorkspace: ChatThreadService["findInWorkspace"] = ( workspaceId: string, threadId: string, @@ -65,6 +78,23 @@ const ensureDefault: ChatThreadService["ensureDefault"] = ( chatRepository.ensureDefaultThreadEffect(workspaceId), ) +const ensureDemo: ChatThreadService["ensureDemo"] = ( + workspaceId: string, + catalog: DemoCatalog, +) => { + const messages = demoView.toChatMessages(catalog) + const firstUserMessage = messages.find((message) => message.role === "user") + if (!firstUserMessage) return Promise.resolve(null) + + return databaseRuntime.runPromise( + chatRepository.ensureDemoThreadEffect(workspaceId, { + demoKey: seededDemoChatKey, + title: firstUserMessage.content, + messages, + }), + ) +} + const listMessages: ChatThreadService["listMessages"] = ( workspaceId: string, threadId: string, @@ -94,6 +124,7 @@ export const chatThreadService: ChatThreadService = { listForWorkspace, create, ensureDefault, + ensureDemo, listMessages, softDelete, appendMessage, diff --git a/src/domains/chunks/index.ts b/src/domains/chunks/index.ts index d3d98ce..d774708 100644 --- a/src/domains/chunks/index.ts +++ b/src/domains/chunks/index.ts @@ -7,7 +7,7 @@ import type { ChatCitationView } from "@/domains/chat/types" import type { ParsedChunkView } from "@/domains/chunks/types" const documentChunkPageSize = 200 -const defaultChunkPageSize = 100 +const defaultChunkPageSize = 50 const maximumChunkPageSize = 200 export type ChunkKnowhereClient = { diff --git a/src/domains/demo/workspace-source-resolution.ts b/src/domains/demo/workspace-source-resolution.ts index e14339f..39abc5a 100644 --- a/src/domains/demo/workspace-source-resolution.ts +++ b/src/domains/demo/workspace-source-resolution.ts @@ -6,6 +6,10 @@ type WorkspaceDemoSourceResolution = { readonly workspaceSources: readonly Source[] } +type SourceViewOptions = { + readonly chunkCount?: number +} + export function resolveWorkspaceDemoSources( sources: readonly Source[], catalog: DemoCatalog, @@ -35,6 +39,32 @@ export function resolveWorkspaceDemoSources( } } +export function getWorkspaceSourcesNeedingKnowhereChunkCount( + sources: readonly Source[], +): Source[] { + return sources.filter((source) => !source.demoKey) +} + +export function getMaterializedDemoSourceViewOptionsBySourceId( + sources: readonly Source[], + catalog: DemoCatalog, +): ReadonlyMap { + const chunkCountByDemoSourceId: ReadonlyMap = new Map( + catalog.sources.map((source) => [source.demoSourceId, source.chunkCount]), + ) + + return new Map( + sources.flatMap((source): readonly [string, SourceViewOptions][] => { + if (!source.demoKey) return [] + + const chunkCount = chunkCountByDemoSourceId.get(source.demoKey) + if (chunkCount === undefined) return [] + + return [[source.id, { chunkCount }]] + }), + ) +} + function isLegacyCanonicalDemoSource( source: Source, canonicalDocumentIdByDemoSourceId: ReadonlyMap, diff --git a/src/domains/sources/background-reconcile.ts b/src/domains/sources/background-reconcile.ts new file mode 100644 index 0000000..3b2a8c6 --- /dev/null +++ b/src/domains/sources/background-reconcile.ts @@ -0,0 +1,70 @@ +import "server-only" + +import { Effect } from "effect" +import { Client } from "@upstash/workflow" + +import { logger } from "@/lib/logger" + +// Re-trigger protection: Layers 1 & 2. +// +// Layer 1 โ€” Upstash idempotency via workflowRunId=sourceId ensures at most one +// running workflow per source, even across process restarts or multiple instances. +// +// Layer 2 โ€” In-memory Set avoids the network call entirely when the same process +// already triggered a workflow for this source. + +const triggeredSourceIds = new Set() + +function createClient(): Client { + return new Client({ token: process.env.QSTASH_TOKEN! }) +} + +function resolveBaseURL(): string { + return process.env.NOTEBOOK_PUBLIC_URL ?? "http://localhost:3000" +} + +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const startBackgroundReconciliationEffect = ( + workspaceId: string, + sourceId: string, + apiKey: string, +): Effect.Effect => + Effect.gen(function* () { + if (triggeredSourceIds.has(sourceId)) return + triggeredSourceIds.add(sourceId) + + yield* Effect.tryPromise(() => + createClient().trigger({ + url: `${resolveBaseURL()}/api/sources/reconcile`, + body: { workspaceId, sourceId, apiKey }, + workflowRunId: sourceId, + retries: 3, + }), + ) + yield* Effect.logInfo( + `background-reconcile: workflow triggered for ${sourceId}`, + ) + }).pipe( + Effect.catchAll((error) => + Effect.sync(() => { + triggeredSourceIds.delete(sourceId) + logger.error("background-reconcile: failed to trigger workflow", { + sourceId, + error: String(error), + }) + }), + ), + ) + +export async function startBackgroundReconciliation( + workspaceId: string, + sourceId: string, + apiKey: string, +): Promise { + return Effect.runPromise( + startBackgroundReconciliationEffect(workspaceId, sourceId, apiKey), + ) +} diff --git a/src/domains/sources/counts.test.ts b/src/domains/sources/counts.test.ts index 816c72f..22cdda9 100644 --- a/src/domains/sources/counts.test.ts +++ b/src/domains/sources/counts.test.ts @@ -76,7 +76,7 @@ describe("countChunksBySourceId", () => { expect(counts.size).toBe(0) }) - it("counts materialized demo sources through their copied document id", async () => { + it("does not count materialized demo sources through their copied document id", async () => { const listChunks = vi.fn().mockResolvedValue({ pagination: { total: 70 }, }) @@ -99,10 +99,7 @@ describe("countChunksBySourceId", () => { ), ) - expect(listChunks).toHaveBeenCalledWith("doc_user_copy", { - page: 1, - pageSize: 1, - }) - expect(counts).toEqual(new Map([["source_demo", 70]])) + expect(listChunks).not.toHaveBeenCalled() + expect(counts.size).toBe(0) }) }) diff --git a/src/domains/sources/counts.ts b/src/domains/sources/counts.ts index 9419714..310a694 100644 --- a/src/domains/sources/counts.ts +++ b/src/domains/sources/counts.ts @@ -11,7 +11,10 @@ export const countChunksBySourceId = ( ) => Effect.gen(function* () { const readySources = sources.filter( - (source) => source.status === "ready" && source.knowhereDocumentId, + (source) => + !source.demoKey && + source.status === "ready" && + source.knowhereDocumentId, ) if (readySources.length === 0) return new Map() diff --git a/src/domains/sources/lifecycle.ts b/src/domains/sources/lifecycle.ts index cbc1404..780dad2 100644 --- a/src/domains/sources/lifecycle.ts +++ b/src/domains/sources/lifecycle.ts @@ -1,5 +1,6 @@ import "server-only" +import { Effect } from "effect" import type { JobResult } from "@ontos-ai/knowhere-sdk" import type { Source } from "@/infrastructure/db/schema" @@ -23,6 +24,7 @@ type SourceLifecycleRepository = { workspaceId: string, sourceId: string, reason: string, + requiredStatus?: string, ): Promise clearSourceStagedBlob(workspaceId: string, sourceId: string): Promise } @@ -47,6 +49,81 @@ type ApplyKnowhereJobToSourceInput = { blobStore: SourceLifecycleBlobStore } +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +export const applyKnowhereJobToSourceEffect = Effect.fn( + "applyKnowhereJobToSource", +)( + function* ({ + workspaceId, + source, + job, + client, + repository, + parsedResultStore, + blobStore, + }: ApplyKnowhereJobToSourceInput) { + // Best-effort early exit: skip expensive asset uploads when the source has + // already been resolved. The atomic guard (Layer 3) is in the DB UPDATE below. + if (source.status !== "parsing") return + + if (job.isDone || job.status === "done") { + if (job.documentId) { + const stored = yield* Effect.tryPromise(() => + parsedResultStore.storeParsedResultAssets({ + workspaceId, + sourceId: source.id, + job, + client, + }), + ) + yield* Effect.tryPromise(() => + repository.saveSourceParseResult(workspaceId, source.id, stored), + ) + yield* Effect.tryPromise(() => + repository.markSourceReady(workspaceId, source.id, job.documentId!), + ) + yield* cleanupStagedBlobEffect( + workspaceId, + source, + repository, + blobStore, + ) + return + } + + yield* Effect.tryPromise(() => + repository.markSourceFailed( + workspaceId, + source.id, + "Parsing finished but no document was published.", + "parsing", + ), + ) + yield* cleanupStagedBlobEffect(workspaceId, source, repository, blobStore) + return + } + + if (job.isFailed || job.status === "failed") { + yield* Effect.tryPromise(() => + repository.markSourceFailed( + workspaceId, + source.id, + job.error?.message ?? "Parsing failed.", + "parsing", + ), + ) + yield* cleanupStagedBlobEffect(workspaceId, source, repository, blobStore) + } + }, +) + +// --------------------------------------------------------------------------- +// Async wrapper (backward-compatible) +// --------------------------------------------------------------------------- + export async function applyKnowhereJobToSource({ workspaceId, source, @@ -56,51 +133,39 @@ export async function applyKnowhereJobToSource({ parsedResultStore, blobStore, }: ApplyKnowhereJobToSourceInput): Promise { - if (job.isDone || job.status === "done") { - if (job.documentId) { - const stored = await parsedResultStore.storeParsedResultAssets({ - workspaceId, - sourceId: source.id, - job, - client, - }) - await repository.saveSourceParseResult(workspaceId, source.id, stored) - await repository.markSourceReady(workspaceId, source.id, job.documentId) - await cleanupStagedBlob(workspaceId, source, repository, blobStore) - return - } - - await repository.markSourceFailed( - workspaceId, - source.id, - "Parsing finished but no document was published.", - ) - await cleanupStagedBlob(workspaceId, source, repository, blobStore) - return - } - - if (job.isFailed || job.status === "failed") { - await repository.markSourceFailed( + return Effect.runPromise( + applyKnowhereJobToSourceEffect({ workspaceId, - source.id, - job.error?.message ?? "Parsing failed.", - ) - await cleanupStagedBlob(workspaceId, source, repository, blobStore) - } + source, + job, + client, + repository, + parsedResultStore, + blobStore, + }), + ) } -async function cleanupStagedBlob( +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function cleanupStagedBlobEffect( workspaceId: string, source: Source, repository: SourceLifecycleRepository, blobStore: SourceLifecycleBlobStore, -): Promise { - if (!source.stagedBlobPathname) return - - try { - await blobStore.deleteStagedSourceBlob(source.stagedBlobPathname) - await repository.clearSourceStagedBlob(workspaceId, source.id) - } catch { - // Staged upload cleanup is best-effort; source state is already advanced. - } +): Effect.Effect { + if (!source.stagedBlobPathname) return Effect.void + + return Effect.gen(function* () { + yield* Effect.tryPromise(() => + blobStore.deleteStagedSourceBlob(source.stagedBlobPathname!), + ) + yield* Effect.tryPromise(() => + repository.clearSourceStagedBlob(workspaceId, source.id), + ) + }).pipe( + Effect.catchAllCause(() => Effect.void), + ) } diff --git a/src/domains/sources/parsed-result-assets.ts b/src/domains/sources/parsed-result-assets.ts index 39ae608..6a36732 100644 --- a/src/domains/sources/parsed-result-assets.ts +++ b/src/domains/sources/parsed-result-assets.ts @@ -2,6 +2,7 @@ import "server-only" import path from "node:path" import { put } from "@vercel/blob" +import { Effect } from "effect" import type { JobResult } from "@ontos-ai/knowhere-sdk" export type StoredParsedResultAssets = { @@ -52,6 +53,73 @@ export type StoreParsedResultAssetsInput = { const parsedResultDirectoryName = "parsed-result" +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +export const storeParsedResultAssetsEffect = Effect.fn( + "storeParsedResultAssets", +)( + function* ({ + workspaceId, + sourceId, + job, + client, + blobStore = vercelBlobStore, + }: StoreParsedResultAssetsInput) { + const parseResult = (yield* Effect.tryPromise(() => + client.jobs.load(job), + )) as ParsedResultWithAssets + const blobPrefix = getParsedResultBlobPrefix(workspaceId, sourceId) + const resultBlob = yield* Effect.tryPromise(() => + blobStore.put( + `${blobPrefix}/result.zip`, + parseResult.rawZip, + getBlobPutOptions("application/zip"), + ), + ) + + const assetUrlsByFilePath: Record = {} + + for (const image of parseResult.imageChunks ?? []) { + const filePath = normalizeParsedAssetPath(image.filePath) + if (!filePath || !image.data) continue + + const blob = yield* Effect.tryPromise(() => + blobStore.put( + `${blobPrefix}/${filePath}`, + image.data!, + getBlobPutOptions(getContentTypeForPath(filePath)), + ), + ) + assetUrlsByFilePath[filePath] = blob.url + } + + for (const table of parseResult.tableChunks ?? []) { + const filePath = normalizeParsedAssetPath(table.filePath) + if (!filePath || typeof table.html !== "string") continue + + const blob = yield* Effect.tryPromise(() => + blobStore.put( + `${blobPrefix}/${filePath}`, + table.html!, + getBlobPutOptions("text/html; charset=utf-8"), + ), + ) + assetUrlsByFilePath[filePath] = blob.url + } + + return { + resultBlobUrl: resultBlob.url, + assetUrlsByFilePath, + } + }, +) + +// --------------------------------------------------------------------------- +// Async wrapper (backward-compatible) +// --------------------------------------------------------------------------- + export async function storeParsedResultAssets({ workspaceId, sourceId, @@ -59,46 +127,21 @@ export async function storeParsedResultAssets({ client, blobStore = vercelBlobStore, }: StoreParsedResultAssetsInput): Promise { - const parseResult = (await client.jobs.load(job)) as ParsedResultWithAssets - const blobPrefix = getParsedResultBlobPrefix(workspaceId, sourceId) - const resultBlob = await blobStore.put( - `${blobPrefix}/result.zip`, - parseResult.rawZip, - getBlobPutOptions("application/zip"), + return Effect.runPromise( + storeParsedResultAssetsEffect({ + workspaceId, + sourceId, + job, + client, + blobStore, + }), ) - - const assetUrlsByFilePath: Record = {} - - for (const image of parseResult.imageChunks ?? []) { - const filePath = normalizeParsedAssetPath(image.filePath) - if (!filePath || !image.data) continue - - const blob = await blobStore.put( - `${blobPrefix}/${filePath}`, - image.data, - getBlobPutOptions(getContentTypeForPath(filePath)), - ) - assetUrlsByFilePath[filePath] = blob.url - } - - for (const table of parseResult.tableChunks ?? []) { - const filePath = normalizeParsedAssetPath(table.filePath) - if (!filePath || typeof table.html !== "string") continue - - const blob = await blobStore.put( - `${blobPrefix}/${filePath}`, - table.html, - getBlobPutOptions("text/html; charset=utf-8"), - ) - assetUrlsByFilePath[filePath] = blob.url - } - - return { - resultBlobUrl: resultBlob.url, - assetUrlsByFilePath, - } } +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + function getParsedResultBlobPrefix( workspaceId: string, sourceId: string, diff --git a/src/domains/sources/reconcile.ts b/src/domains/sources/reconcile.ts index 3be1954..e71ea9a 100644 --- a/src/domains/sources/reconcile.ts +++ b/src/domains/sources/reconcile.ts @@ -1,10 +1,11 @@ import "server-only" +import { Effect, pipe } from "effect" import { del } from "@vercel/blob" import type Knowhere from "@ontos-ai/knowhere-sdk" import type { JobResult } from "@ontos-ai/knowhere-sdk" -import type { Source, Workspace } from "@/infrastructure/db/schema" +import type { Source } from "@/infrastructure/db/schema" import { storeParsedResultAssets, type StoreParsedResultAssetsInput, @@ -29,34 +30,65 @@ type SourceReconcileDependencies = { ) => Promise } +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +export const reconcileSourcesForWorkspaceEffect = Effect.fn( + "reconcileSourcesForWorkspace", +)( + function* ( + workspace: { readonly id: string }, + client: Knowhere, + deps: SourceReconcileDependencies = {}, + ) { + const rows = yield* Effect.tryPromise(() => + sourceWorkflowRuntime.listForWorkspace(workspace.id), + ) + const parsing = rows.filter( + (row) => row.status === "parsing" && row.knowhereJobId, + ) + if (parsing.length === 0) return rows + + yield* pipe( + parsing, + Effect.forEach( + (source) => + Effect.gen(function* () { + const jobId = source.knowhereJobId! + const job = yield* Effect.tryPromise(() => client.jobs.get(jobId)) + yield* Effect.tryPromise(() => + updateSourceFromJob(workspace.id, source, job, client, deps), + ) + }).pipe(Effect.catchAllCause(() => Effect.void)), + { concurrency: "unbounded" }, + ), + ) + + return yield* Effect.tryPromise(() => + sourceWorkflowRuntime.listForWorkspace(workspace.id), + ) + }, +) + +// --------------------------------------------------------------------------- +// Async wrapper (backward-compatible) +// --------------------------------------------------------------------------- + export async function reconcileSourcesForWorkspace( - workspace: Workspace, + workspace: { readonly id: string }, client: Knowhere, deps: SourceReconcileDependencies = {}, ): Promise { - const rows = await sourceWorkflowRuntime.listForWorkspace(workspace.id) - const parsing = rows.filter( - (row) => row.status === "parsing" && row.knowhereJobId, + return Effect.runPromise( + reconcileSourcesForWorkspaceEffect(workspace, client, deps), ) - if (parsing.length === 0) return rows - - await Promise.all( - parsing.map(async (source) => { - const jobId = source.knowhereJobId - if (!jobId) return - - try { - const job = await client.jobs.get(jobId) - await updateSourceFromJob(workspace.id, source, job, client, deps) - } catch { - // Leave the current row as-is on transient API errors. - } - }), - ) - - return await sourceWorkflowRuntime.listForWorkspace(workspace.id) } +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + async function updateSourceFromJob( workspaceId: string, source: Source, diff --git a/src/domains/sources/route-archive.ts b/src/domains/sources/route-archive.ts index 5a85df5..2b570b6 100644 --- a/src/domains/sources/route-archive.ts +++ b/src/domains/sources/route-archive.ts @@ -1,3 +1,5 @@ +import { Effect } from "effect" + import { routeResult } from "@/lib/route-result" import { getClientForWorkspace } from "./route-dependencies" import type { @@ -26,56 +28,67 @@ type RouteArchive = { function createRouteArchive(deps: RouteArchiveDependencies): RouteArchive { return { - archiveSource: (input: ArchiveSourceInput) => archiveSource(input, deps), + archiveSource: (input: ArchiveSourceInput) => + Effect.runPromise(archiveSourceEffect(input, deps)), } } -async function archiveSource( +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const archiveSourceEffect = ( input: ArchiveSourceInput, deps: RouteArchiveDependencies, -): Promise> { - const user = await deps.requireUser() - const workspace = await deps.ensureWorkspace(user.id) - const source = await deps.sourceService.findInWorkspace( - workspace.id, - input.sourceId, - ) - - if (!source) { - const catalog = await deps.demoApi.fetchCatalog() - const isDemoSource = catalog.sources.some( - (candidate) => candidate.demoSourceId === input.sourceId, +) => + Effect.gen(function* () { + const user = yield* Effect.tryPromise(() => deps.requireUser()) + const workspace = yield* Effect.tryPromise(() => + deps.ensureWorkspace(user.id), + ) + const source = yield* Effect.tryPromise(() => + deps.sourceService.findInWorkspace(workspace.id, input.sourceId), ) - if (isDemoSource) { - await deps.sourceService.hideDemoSource(workspace.id, input.sourceId) - return routeResult.ok({ id: input.sourceId, archived: true }) + + if (!source) { + const catalog = yield* Effect.tryPromise(() => deps.demoApi.fetchCatalog()) + const isDemoSource = catalog.sources.some( + (candidate) => candidate.demoSourceId === input.sourceId, + ) + if (isDemoSource) { + yield* Effect.tryPromise(() => + deps.sourceService.hideDemoSource(workspace.id, input.sourceId), + ) + return routeResult.ok({ id: input.sourceId, archived: true as const }) + } + + return routeResult.error(404, "Source not found.") } - return routeResult.error(404, "Source not found.") - } + if (source.knowhereDocumentId) { + const client = yield* Effect.tryPromise(() => + getClientForWorkspace(workspace.id, input.cookieHeader, deps), + ) + yield* Effect.tryPromise(() => + client.documents.archive(source.knowhereDocumentId!), + ) + } - if (source.knowhereDocumentId) { - const client = await getClientForWorkspace( - workspace.id, - input.cookieHeader, - deps, + yield* Effect.tryPromise(() => + deps.sourceService.softDelete(workspace.id, input.sourceId), ) - await client.documents.archive(source.knowhereDocumentId) - } - - await deps.sourceService.softDelete(workspace.id, input.sourceId) - if (source.demoKey) { - await deps.sourceService.hideDemoSource(workspace.id, source.demoKey) - } - if (source.originalBlobPathname) { - try { - await deps.deleteBlob(source.originalBlobPathname) - } catch { - // Source archival already succeeded; Blob cleanup is best-effort. + if (source.demoKey) { + yield* Effect.tryPromise(() => + deps.sourceService.hideDemoSource(workspace.id, source.demoKey!), + ) + } + if (source.originalBlobPathname) { + yield* Effect.tryPromise(() => + deps.deleteBlob(source.originalBlobPathname!), + ).pipe(Effect.catchAllCause(() => Effect.void)) } - } - return routeResult.ok({ id: input.sourceId, archived: true }) -} + return routeResult.ok({ id: input.sourceId, archived: true as const }) + }) export { createRouteArchive } diff --git a/src/domains/sources/route-chunks.ts b/src/domains/sources/route-chunks.ts index 17c67dd..f85ad34 100644 --- a/src/domains/sources/route-chunks.ts +++ b/src/domains/sources/route-chunks.ts @@ -2,8 +2,10 @@ import { Effect } from "effect" import { demoView } from "@/domains/demo/view" import type { DemoChunkPage } from "@/integrations/knowhere-demo" +import { logger } from "@/lib/logger" import { routeResult } from "@/lib/route-result" import { getClientForWorkspace } from "./route-dependencies" +import { sourceRowRepository } from "./source-row-repository" import type { JsonRouteResult, LoadSourceChunksInput, @@ -32,67 +34,94 @@ type RouteChunks = { function createRouteChunks(deps: RouteChunksDependencies): RouteChunks { return { loadSourceChunks: (input: LoadSourceChunksInput) => - loadSourceChunks(input, deps), + Effect.runPromise(loadSourceChunksEffect(input, deps)), } } -async function loadSourceChunks( +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const loadSourceChunksEffect = ( input: LoadSourceChunksInput, deps: RouteChunksDependencies, -): Promise> { - const user = await deps.getCurrentUser() - if (!user) { - return (await loadDemoChunkPage(input, deps)) ?? sourceNotFound() - } +) => + Effect.gen(function* () { + if (!sourceRowRepository.isWorkspaceSourceId(input.sourceId)) { + const demoResult = yield* loadDemoChunkPageEffect(input, deps) + return demoResult ?? sourceNotFound() + } - const workspace = await deps.ensureWorkspace(user.id) - const source = await deps.sourceService.findInWorkspace( - workspace.id, - input.sourceId, - ) + const user = yield* Effect.tryPromise(() => deps.getCurrentUser()) + if (!user) { + const demoResult = yield* loadDemoChunkPageEffect(input, deps) + return demoResult ?? sourceNotFound() + } - if (!source) { - return (await loadDemoChunkPage(input, deps)) ?? sourceNotFound() - } + const workspace = yield* Effect.tryPromise(() => + deps.ensureWorkspace(user.id), + ) + const source = yield* Effect.tryPromise(() => + deps.sourceService.findInWorkspace(workspace.id, input.sourceId), + ) - const client = await getClientForWorkspace( - workspace.id, - input.cookieHeader, - deps, - ) - const assetUrlsByFilePath = await deps.sourceService.getParseAssetUrls( - workspace.id, - source.id, - ) + if (!source) { + const demoResult = yield* loadDemoChunkPageEffect(input, deps) + return demoResult ?? sourceNotFound() + } - if (input.shouldLoadAll) { - const chunks = await Effect.runPromise( - deps.loadChunksForSource(source, client, { assetUrlsByFilePath }), + if (source.demoKey) { + const demoResult = yield* loadDemoChunkPageEffect( + input, + deps, + source.demoKey, + source.knowhereDocumentId, + ) + return demoResult ?? sourceNotFound() + } + + const client = yield* Effect.tryPromise(() => + getClientForWorkspace(workspace.id, input.cookieHeader, deps), + ) + const assetUrlsByFilePath = yield* Effect.tryPromise(() => + deps.sourceService.getParseAssetUrls(workspace.id, source.id), ) - return routeResult.ok({ chunks }) - } - const chunkPage = await Effect.runPromise( - deps.loadChunkPageForSource(source, client, input.pageParams, { - assetUrlsByFilePath, - }), - ) - return routeResult.ok(chunkPage) -} + if (input.shouldLoadAll) { + const chunks = yield* deps.loadChunksForSource(source, client, { + assetUrlsByFilePath, + }) + return routeResult.ok({ chunks }) + } + + const chunkPage = yield* deps.loadChunkPageForSource( + source, + client, + input.pageParams, + { assetUrlsByFilePath }, + ) + return routeResult.ok(chunkPage) + }) -async function loadDemoChunkPage( +const loadDemoChunkPageEffect = ( input: LoadSourceChunksInput, deps: RouteChunksDependencies, -): Promise | null> { - try { + demoSourceId: string = input.sourceId, + documentIdOverride?: string | null, +) => + Effect.gen(function* () { const pages = input.shouldLoadAll - ? await loadAllDemoChunkPages(input, deps) + ? yield* Effect.tryPromise(() => + loadAllDemoChunkPages(input, deps, demoSourceId), + ) : [ - await deps.demoApi.fetchChunkPage({ - demoSourceId: input.sourceId, - page: input.pageParams.page, - pageSize: input.pageParams.pageSize, - }), + yield* Effect.tryPromise(() => + deps.demoApi.fetchChunkPage({ + demoSourceId, + page: input.pageParams.page, + pageSize: input.pageParams.pageSize, + }), + ), ] const page = pages[0] if (!page) return null @@ -103,7 +132,7 @@ async function loadDemoChunkPage( title: page.title, mimeType: page.mimeType, status: "ready" as const, - documentId: page.canonicalDocumentId, + documentId: documentIdOverride ?? page.canonicalDocumentId, } const chunks = pages.flatMap((demoChunkPage) => demoChunkPage.chunks.map((chunk) => @@ -119,18 +148,31 @@ async function loadDemoChunkPage( pagination: page.pagination, }, ) - } catch { - return null - } -} + }).pipe( + Effect.catchAll((error) => + Effect.sync(() => { + logger.warn("sources: demo chunk load failed", { + sourceId: input.sourceId, + demoSourceId, + page: input.pageParams.page, + pageSize: input.pageParams.pageSize, + shouldLoadAll: input.shouldLoadAll, + knowhereBaseUrl: process.env.KNOWHERE_BASE_URL ?? "(default)", + error: getErrorMessage(error), + }) + return null + }), + ), + ) async function loadAllDemoChunkPages( input: LoadSourceChunksInput, deps: RouteChunksDependencies, + demoSourceId: string, ): Promise { const pageSize = 200 const firstPage = await deps.demoApi.fetchChunkPage({ - demoSourceId: input.sourceId, + demoSourceId, page: 1, pageSize, }) @@ -142,7 +184,7 @@ async function loadAllDemoChunkPages( ) { pages.push( await deps.demoApi.fetchChunkPage({ - demoSourceId: input.sourceId, + demoSourceId, page: pageNumber, pageSize, }), @@ -151,6 +193,14 @@ async function loadAllDemoChunkPages( return pages } +function getErrorMessage(error: unknown): string { + if (error instanceof Error) { + const inner = (error as Error & { error?: unknown }).error + return inner instanceof Error ? inner.message : error.message + } + return String(error) +} + function sourceNotFound(): JsonRouteResult<{ readonly message: string }> { return routeResult.error(404, "Source not found.") } diff --git a/src/domains/sources/route-dependencies.ts b/src/domains/sources/route-dependencies.ts index 54bf0c2..0af5abb 100644 --- a/src/domains/sources/route-dependencies.ts +++ b/src/domains/sources/route-dependencies.ts @@ -13,6 +13,7 @@ import { getCurrentUser, requireUser } from "@/infrastructure/auth" import { workspaceService } from "@/domains/workspace/service" import { sourceViewOptionsBySourceId as getDefaultSourceViewOptionsBySourceId } from "./counts" import { reconcileSourcesForWorkspace as reconcileDefaultSourcesForWorkspace } from "./reconcile" +import { sourceWorkflowRuntime } from "./workflow-runtime" import { sourceService as defaultSourceService } from "./service" import type { SourceRouteKnowhereClient, @@ -35,6 +36,7 @@ const defaultDependencies: SourceRouteServiceDependencies = { loadChunksForSource, makeKnowhereClient: (apiKey: string) => makeDefaultKnowhereClient(apiKey) as SourceRouteKnowhereClient, + listSourcesForWorkspace: sourceWorkflowRuntime.listForWorkspace, reconcileSourcesForWorkspace: (workspace, client) => reconcileDefaultSourcesForWorkspace( workspace, diff --git a/src/domains/sources/route-listing.ts b/src/domains/sources/route-listing.ts index 95adbb3..b204969 100644 --- a/src/domains/sources/route-listing.ts +++ b/src/domains/sources/route-listing.ts @@ -1,11 +1,15 @@ import { Effect } from "effect" import { demoView } from "@/domains/demo/view" -import { resolveWorkspaceDemoSources } from "@/domains/demo/workspace-source-resolution" +import { + getMaterializedDemoSourceViewOptionsBySourceId, + getWorkspaceSourcesNeedingKnowhereChunkCount, + resolveWorkspaceDemoSources, +} from "@/domains/demo/workspace-source-resolution" import { routeResult } from "@/lib/route-result" -import type { DemoCatalog } from "@/integrations/knowhere-demo" +import { knowhereDemoApi } from "@/integrations/knowhere-demo" import { toSourceView } from "./view" -import { getClientForWorkspace } from "./route-dependencies" +import { startBackgroundReconciliation } from "./background-reconcile" import type { JsonRouteResult, ListSourcesBody, @@ -19,10 +23,13 @@ type RouteListingDependencies = Pick< | "ensureWorkspace" | "getCurrentUser" | "getSourceViewOptionsBySourceId" + | "listSourcesForWorkspace" | "makeKnowhereClient" - | "reconcileSourcesForWorkspace" > & { - readonly demoApi: Pick + readonly demoApi: Pick< + SourceRouteServiceDependencies["demoApi"], + "fetchCatalog" + > readonly sourceService: Pick< SourceRouteServiceDependencies["sourceService"], "listHiddenDemoSourceIds" @@ -35,68 +42,93 @@ type RouteListing = { ) => Promise> } -const emptyDemoCatalog: DemoCatalog = { sources: [] } - function createRouteListing(deps: RouteListingDependencies): RouteListing { return { - listSources: (input: ListSourcesInput) => listSources(input, deps), + listSources: (input: ListSourcesInput) => + Effect.runPromise(listSourcesEffect(input, deps)), } } -async function listSources( +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const listSourcesEffect = ( input: ListSourcesInput, deps: RouteListingDependencies, -): Promise> { - const user = await deps.getCurrentUser() - if (!user) { - const catalog = await deps.demoApi.fetchCatalog() - return routeResult.ok({ sources: catalog.sources.map(demoView.toSourceView) }) - } +) => + Effect.gen(function* () { + const user = yield* Effect.tryPromise(() => deps.getCurrentUser()) + if (!user) { + const catalog = yield* Effect.tryPromise(() => deps.demoApi.fetchCatalog()) + return routeResult.ok({ + sources: catalog.sources.map(demoView.toSourceView), + }) + } - const catalog = await fetchOptionalDemoCatalog(deps.demoApi.fetchCatalog) - const workspace = await deps.ensureWorkspace(user.id) - const client = await getClientForWorkspace( - workspace.id, - input.cookieHeader, - deps, - ) - const sources = await deps.reconcileSourcesForWorkspace(workspace, client) - const demoSourceResolution = resolveWorkspaceDemoSources(sources, catalog) - const sourceOptions = await Effect.runPromise( - deps.getSourceViewOptionsBySourceId( - demoSourceResolution.workspaceSources, + const catalog = yield* Effect.tryPromise(() => + knowhereDemoApi.fetchOptionalCatalog(deps.demoApi.fetchCatalog), + ) + const workspace = yield* Effect.tryPromise(() => + deps.ensureWorkspace(user.id), + ) + const sources = yield* Effect.tryPromise(() => + deps.listSourcesForWorkspace(workspace.id), + ) + const demoSourceResolution = resolveWorkspaceDemoSources(sources, catalog) + const sourcesNeedingKnowhereChunkCount = + getWorkspaceSourcesNeedingKnowhereChunkCount( + demoSourceResolution.workspaceSources, + ) + const materializedDemoSourceOptions = + getMaterializedDemoSourceViewOptionsBySourceId( + demoSourceResolution.workspaceSources, + catalog, + ) + const apiKey = yield* Effect.tryPromise(() => + deps.ensureApiKeyForWorkspace(workspace.id, input.cookieHeader), + ) + const client = deps.makeKnowhereClient(apiKey) + for (const source of sources) { + if (source.status === "parsing" && source.knowhereJobId) { + yield* Effect.fork( + Effect.tryPromise(() => + startBackgroundReconciliation(workspace.id, source.id, apiKey), + ), + ) + } + } + const sourceOptions = yield* deps.getSourceViewOptionsBySourceId( + sourcesNeedingKnowhereChunkCount, client, - ), - ) - const hiddenDemoSourceIds = new Set( - await deps.sourceService.listHiddenDemoSourceIds(workspace.id), - ) - const visibleDemoSources = catalog.sources - .filter( - (source) => - !demoSourceResolution.materializedDemoSourceIds.has(source.demoSourceId), ) - .filter((source) => !hiddenDemoSourceIds.has(source.demoSourceId)) - .map(demoView.toSourceView) - - return routeResult.ok({ - sources: [ - ...visibleDemoSources, - ...demoSourceResolution.workspaceSources.map((source) => - toSourceView(source, sourceOptions.get(source.id)), + const hiddenDemoSourceIds = new Set( + yield* Effect.tryPromise(() => + deps.sourceService.listHiddenDemoSourceIds(workspace.id), ), - ], - }) -} + ) + const visibleDemoSources = catalog.sources + .filter( + (source) => + !demoSourceResolution.materializedDemoSourceIds.has( + source.demoSourceId, + ), + ) + .filter((source) => !hiddenDemoSourceIds.has(source.demoSourceId)) + .map(demoView.toSourceView) -async function fetchOptionalDemoCatalog( - fetchDemoCatalog: () => Promise, -): Promise { - try { - return await fetchDemoCatalog() - } catch { - return emptyDemoCatalog - } -} + return routeResult.ok({ + sources: [ + ...visibleDemoSources, + ...demoSourceResolution.workspaceSources.map((source) => + toSourceView( + source, + materializedDemoSourceOptions.get(source.id) ?? + sourceOptions.get(source.id), + ), + ), + ], + }) + }) export { createRouteListing } diff --git a/src/domains/sources/route-service.test.ts b/src/domains/sources/route-service.test.ts index abaa06c..650fce1 100644 --- a/src/domains/sources/route-service.test.ts +++ b/src/domains/sources/route-service.test.ts @@ -58,7 +58,7 @@ describe("source route service", () => { const getSourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map([[source.id, { chunkCount: 8 }]])), ); - const reconcileSourcesForWorkspace = vi.fn(async () => [source]); + const listSourcesForWorkspace = vi.fn(async () => [source]); const listHiddenDemoSourceIds = vi.fn(async () => []); const listing = createRouteListing({ demoApi: { @@ -73,7 +73,7 @@ describe("source route service", () => { })), getSourceViewOptionsBySourceId, makeKnowhereClient: vi.fn(() => knowhereClient), - reconcileSourcesForWorkspace, + listSourcesForWorkspace, sourceService: { listHiddenDemoSourceIds }, }); @@ -99,10 +99,7 @@ describe("source route service", () => { workspace.id, "session=abc", ); - expect(reconcileSourcesForWorkspace).toHaveBeenCalledWith( - workspace, - knowhereClient, - ); + expect(listSourcesForWorkspace).toHaveBeenCalledWith(workspace.id); expect(listHiddenDemoSourceIds).toHaveBeenCalledWith(workspace.id); }); @@ -151,7 +148,7 @@ describe("source route service", () => { })), getSourceViewOptionsBySourceId, makeKnowhereClient: vi.fn(() => knowhereClient), - reconcileSourcesForWorkspace: vi.fn(async () => [legacyFakeSource, source]), + listSourcesForWorkspace: vi.fn(async () => [legacyFakeSource, source]), sourceService: { listHiddenDemoSourceIds: vi.fn(async () => []) }, }); @@ -220,7 +217,7 @@ describe("source route service", () => { })), getSourceViewOptionsBySourceId, makeKnowhereClient: vi.fn(() => knowhereClient), - reconcileSourcesForWorkspace: vi.fn(async () => [legacyFakeSource]), + listSourcesForWorkspace: vi.fn(async () => [legacyFakeSource]), sourceService: { listHiddenDemoSourceIds: vi.fn(async () => []) }, }); @@ -296,7 +293,7 @@ describe("source route service", () => { })), getSourceViewOptionsBySourceId, makeKnowhereClient: vi.fn(() => knowhereClient), - reconcileSourcesForWorkspace: vi.fn(async () => [nonReadyLegacySource]), + listSourcesForWorkspace: vi.fn(async () => [nonReadyLegacySource]), sourceService: { listHiddenDemoSourceIds: vi.fn(async () => []) }, }); @@ -320,6 +317,75 @@ describe("source route service", () => { }); }); + it("uses demo catalog counts for materialized demo sources", async () => { + const materializedSource: Source = { + ...source, + id: "source_demo", + title: "TSLA-Q4-2025-Update.pdf", + status: "ready", + demoKey: "demo-tsla-q4-2025", + knowhereJobId: null, + knowhereDocumentId: "doc_user_copy", + originalBlobUrl: "/api/demo-sources/demo-tsla-q4-2025/original", + }; + const knowhereClient = { + documents: { + archive: vi.fn(async () => undefined), + listChunks: vi.fn(async () => ({ + chunks: [], + pagination: { + page: 1, + pageSize: 1, + total: 0, + totalPages: 0, + }, + })), + }, + jobs: { + create: vi.fn(), + upload: vi.fn(), + }, + }; + const getSourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map())); + const listing = createRouteListing({ + demoApi: { + fetchCatalog: vi.fn(async () => demoCatalog), + }, + ensureApiKeyForWorkspace: vi.fn(async () => "jwt_123"), + ensureWorkspace: vi.fn(async () => workspace), + getCurrentUser: vi.fn(async () => ({ + id: "user_1", + email: null, + name: null, + })), + getSourceViewOptionsBySourceId, + makeKnowhereClient: vi.fn(() => knowhereClient), + listSourcesForWorkspace: vi.fn(async () => [materializedSource]), + sourceService: { listHiddenDemoSourceIds: vi.fn(async () => []) }, + }); + + const result = await listing.listSources({ cookieHeader: "session=abc" }); + + expect(getSourceViewOptionsBySourceId).toHaveBeenCalledWith( + [], + knowhereClient, + ); + expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled(); + expect(result).toEqual({ + status: 200, + body: { + sources: [ + expect.objectContaining({ + id: "source_demo", + kind: "workspace", + documentId: "doc_user_copy", + chunkCount: 70, + }), + ], + }, + }); + }); + it("lists API-owned demo sources for anonymous users", async () => { const ensureWorkspace = vi.fn(async () => workspace); const service = createSourceRouteService({ diff --git a/src/domains/sources/route-types.ts b/src/domains/sources/route-types.ts index e1e831d..f6e9dad 100644 --- a/src/domains/sources/route-types.ts +++ b/src/domains/sources/route-types.ts @@ -174,6 +174,7 @@ type SourceRouteServiceDependencies = { readonly loadChunkPageForSource: typeof loadChunkPageForSource readonly loadChunksForSource: typeof loadChunksForSource readonly makeKnowhereClient: (apiKey: string) => SourceRouteKnowhereClient + readonly listSourcesForWorkspace: (workspaceId: string) => Promise readonly reconcileSourcesForWorkspace: ( workspace: Workspace, client: SourceRouteKnowhereClient, diff --git a/src/domains/sources/route-upload.ts b/src/domains/sources/route-upload.ts index 6150f47..2f6f7f4 100644 --- a/src/domains/sources/route-upload.ts +++ b/src/domains/sources/route-upload.ts @@ -1,7 +1,9 @@ +import { Effect } from "effect" + import type { Source, Workspace } from "@/infrastructure/db/schema" import { routeResult } from "@/lib/route-result" +import { startBackgroundReconciliation } from "./background-reconcile" import { validateSourceBlobUploadInput } from "./blob-upload" -import { getClientForWorkspace } from "./route-dependencies" import type { JsonRouteResult, SourceRouteKnowhereClient, @@ -30,58 +32,81 @@ type RouteUpload = { function createRouteUpload(deps: RouteUploadDependencies): RouteUpload { return { - uploadSource: (input: UploadSourceInput) => uploadSource(input, deps), + uploadSource: (input: UploadSourceInput) => + Effect.runPromise(uploadSourceEffect(input, deps)), } } -async function uploadSource( +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const uploadSourceEffect = ( input: UploadSourceInput, deps: RouteUploadDependencies, -): Promise> { - const user = await deps.getCurrentUser() - if (!user) { - return routeResult.error(401, "Please log in to upload documents.") - } +) => + Effect.gen(function* () { + const user = yield* Effect.tryPromise(() => deps.getCurrentUser()) + if (!user) { + return routeResult.error(401, "Please log in to upload documents.") + } - if (input.upload.type === "error") { - return routeResult.badRequest(input.upload.message) - } + if (input.upload.type === "error") { + return routeResult.badRequest(input.upload.message) + } - const validation = - input.upload.type === "file" - ? validateUploadFile(input.upload.file) - : validateSourceBlobUploadInput(input.upload.input) - if (!validation.ok) { - return routeResult.badRequest(validation.message) - } + const validation = + input.upload.type === "file" + ? validateUploadFile(input.upload.file) + : validateSourceBlobUploadInput(input.upload.input) + if (!validation.ok) { + return routeResult.badRequest(validation.message) + } - const workspace = await deps.ensureWorkspace(user.id) - const client = await getClientForWorkspace( - workspace.id, - input.cookieHeader, - deps, - ) - const source = await uploadToKnowhere(workspace, input.upload, client, deps) - .finally(() => { - input.onUploadFinished?.() - }) + const workspace = yield* Effect.tryPromise(() => + deps.ensureWorkspace(user.id), + ) + const apiKey = yield* Effect.tryPromise(() => + deps.ensureApiKeyForWorkspace(workspace.id, input.cookieHeader), + ) + const client = deps.makeKnowhereClient(apiKey) - return routeResult.ok({ source: toSourceView(source) }, 201) -} + const source = yield* uploadToKnowhereEffect( + workspace, + input.upload, + client, + deps, + ).pipe( + Effect.onExit(() => + Effect.sync(() => { + input.onUploadFinished?.() + }), + ), + ) + + yield* Effect.tryPromise(() => + startBackgroundReconciliation(workspace.id, source.id, apiKey), + ) + + return routeResult.ok({ source: toSourceView(source) }, 201) + }) -async function uploadToKnowhere( +const uploadToKnowhereEffect = ( workspace: Workspace, upload: Exclude, client: SourceRouteKnowhereClient, deps: RouteUploadDependencies, -): Promise { - return upload.type === "file" - ? deps.sourceService.uploadSourceToKnowhere(workspace, upload.file, client) - : deps.sourceService.uploadSourceBlobToKnowhere( - workspace, - upload.input, - client, +) => + upload.type === "file" + ? Effect.tryPromise(() => + deps.sourceService.uploadSourceToKnowhere(workspace, upload.file, client), + ) + : Effect.tryPromise(() => + deps.sourceService.uploadSourceBlobToKnowhere( + workspace, + upload.input, + client, + ), ) -} export { createRouteUpload } diff --git a/src/domains/sources/source-row-repository.test.ts b/src/domains/sources/source-row-repository.test.ts new file mode 100644 index 0000000..aaa2cce --- /dev/null +++ b/src/domains/sources/source-row-repository.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest" +import { Effect } from "effect" + +import { sourceRowRepository } from "./source-row-repository" +import { DbClient, type Db } from "@/infrastructure/db" + +describe("sourceRowRepository", () => { + it("classifies canonical demo ids as non-workspace source ids", () => { + expect(sourceRowRepository.isWorkspaceSourceId("demo-tsla-q4-2025")).toBe( + false, + ) + }) + + it("classifies UUIDs as workspace source ids", () => { + expect( + sourceRowRepository.isWorkspaceSourceId( + "f03b2dd5-cbc6-44a1-a5cb-8106f8ce52bb", + ), + ).toBe(true) + }) + + it("does not update the database for canonical demo ids", async () => { + const db = makeThrowingDb() + + await expect( + sourceRowRepository.updateInWorkspaceWithDb( + db, + "workspace_1", + "demo-tsla-q4-2025", + { status: "ready" }, + ), + ).resolves.toBeNull() + }) + + it("does not soft-delete the database for canonical demo ids", async () => { + const db = makeThrowingDb() + + await expect( + Effect.runPromise( + sourceRowRepository + .softDeleteEffect("workspace_1", "demo-tsla-q4-2025") + .pipe(Effect.provideService(DbClient, db)), + ), + ).resolves.toBe(false) + }) +}) + +function makeThrowingDb(): Db { + return { + update: () => { + throw new Error("database should not be called for demo source ids") + }, + } as unknown as Db +} diff --git a/src/domains/sources/source-row-repository.ts b/src/domains/sources/source-row-repository.ts index 1ca5178..71b054a 100644 --- a/src/domains/sources/source-row-repository.ts +++ b/src/domains/sources/source-row-repository.ts @@ -59,6 +59,7 @@ type SourceRowRepository = { workspaceId: string, sourceId: string, reason: string, + requiredStatus?: string, ) => Effect.Effect readonly clearStagedBlobEffect: ( workspaceId: string, @@ -68,6 +69,7 @@ type SourceRowRepository = { workspaceId: string, sourceId: string, ) => Effect.Effect + readonly isWorkspaceSourceId: (sourceId: string) => boolean readonly findInWorkspaceWithDb: ( db: Db, workspaceId: string, @@ -78,6 +80,7 @@ type SourceRowRepository = { workspaceId: string, sourceId: string, values: SourceUpdate, + requiredStatus?: string, ) => Promise readonly requireSource: (source: Source | null, message: string) => Source } @@ -161,17 +164,18 @@ const markReadyEffect: SourceRowRepository["markReadyEffect"] = ( status: "ready", knowhereDocumentId: documentId, failureReason: null, - }) + }, "parsing") const markFailedEffect: SourceRowRepository["markFailedEffect"] = ( workspaceId: string, sourceId: string, reason: string, + requiredStatus?: string, ) => updateInWorkspaceEffect(workspaceId, sourceId, { status: "failed", failureReason: reason, - }) + }, requiredStatus) const clearStagedBlobEffect: SourceRowRepository["clearStagedBlobEffect"] = ( workspaceId: string, @@ -187,6 +191,8 @@ const softDeleteEffect: SourceRowRepository["softDeleteEffect"] = ( sourceId: string, ) => Effect.gen(function* () { + if (!isWorkspaceSourceId(sourceId)) return false + const db = yield* DbClient const result = yield* Effect.promise(() => db @@ -209,11 +215,12 @@ const updateInWorkspaceEffect = ( workspaceId: string, sourceId: string, values: SourceUpdate, + requiredStatus?: string, ) => Effect.gen(function* () { const db = yield* DbClient return yield* Effect.promise(() => - updateInWorkspaceWithDb(db, workspaceId, sourceId, values), + updateInWorkspaceWithDb(db, workspaceId, sourceId, values, requiredStatus), ) }) @@ -222,6 +229,8 @@ async function findInWorkspaceWithDb( workspaceId: string, sourceId: string, ): Promise { + if (!isWorkspaceSourceId(sourceId)) return null + const row = await db .select() .from(sources) @@ -242,22 +251,38 @@ async function updateInWorkspaceWithDb( workspaceId: string, sourceId: string, values: SourceUpdate, + requiredStatus?: string, ): Promise { + if (!isWorkspaceSourceId(sourceId)) return null + + // Layer 3 โ€” Atomic status guard. + // When requiredStatus is set, the UPDATE only matches if the source is still in + // the expected status. Two concurrent workflows will race; only one wins. + const conditions = [ + eq(sources.id, sourceId), + eq(sources.workspaceId, workspaceId), + isNull(sources.deletedAt), + ] + if (requiredStatus) { + conditions.push(eq(sources.status, requiredStatus)) + } + const [source] = await db .update(sources) .set({ ...values, updatedAt: sql`now()` }) - .where( - and( - eq(sources.id, sourceId), - eq(sources.workspaceId, workspaceId), - isNull(sources.deletedAt), - ), - ) + .where(and(...conditions)) .returning() return source ?? null } +const WORKSPACE_SOURCE_ID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu + +function isWorkspaceSourceId(sourceId: string): boolean { + return WORKSPACE_SOURCE_ID_PATTERN.test(sourceId) +} + function requireSource(source: Source | null, message: string): Source { if (!source) throw new Error(message) return source @@ -272,6 +297,7 @@ export const sourceRowRepository: SourceRowRepository = { markFailedEffect, clearStagedBlobEffect, softDeleteEffect, + isWorkspaceSourceId, findInWorkspaceWithDb, updateInWorkspaceWithDb, requireSource, diff --git a/src/domains/sources/workflow-runtime.ts b/src/domains/sources/workflow-runtime.ts index b713f58..fff2202 100644 --- a/src/domains/sources/workflow-runtime.ts +++ b/src/domains/sources/workflow-runtime.ts @@ -31,6 +31,7 @@ type UploadRepositoryRuntime = { workspaceId: string, sourceId: string, reason: string, + requiredStatus?: string, ) => Promise } @@ -133,9 +134,10 @@ const markFailed: SourceWorkflowRuntime["markFailed"] = ( workspaceId: string, sourceId: string, reason: string, + requiredStatus?: string, ) => databaseRuntime.runPromise( - sourceRepository.markFailedEffect(workspaceId, sourceId, reason), + sourceRepository.markFailedEffect(workspaceId, sourceId, reason, requiredStatus), ) const clearStagedBlob: SourceWorkflowRuntime["clearStagedBlob"] = ( diff --git a/src/domains/workspace/client.ts b/src/domains/workspace/client.ts index e92d56a..1cea55a 100644 --- a/src/domains/workspace/client.ts +++ b/src/domains/workspace/client.ts @@ -16,7 +16,7 @@ const workspaceClientKeys = { } as const const workspaceClientConfig = { - sourceChunkPageSize: 100, + sourceChunkPageSize: 50, } as const type SourceChunksResponse = { diff --git a/src/domains/workspace/initial-state.test.ts b/src/domains/workspace/initial-state.test.ts index fba6767..322f40c 100644 --- a/src/domains/workspace/initial-state.test.ts +++ b/src/domains/workspace/initial-state.test.ts @@ -3,7 +3,12 @@ import { afterEach, describe, expect, it, vi } from "vitest" import { loadWorkspaceShellInitialState } from "./initial-state" import type { AuthUser } from "@/infrastructure/auth" -import type { ChatThread, Source, Workspace } from "@/infrastructure/db/schema" +import type { + ChatMessage, + ChatThread, + Source, + Workspace, +} from "@/infrastructure/db/schema" import type { DemoCatalog } from "@/integrations/knowhere-demo" type InitialStateDependencies = NonNullable< @@ -76,7 +81,7 @@ describe("loadWorkspaceShellInitialState", () => { }, ]) expect(state.loginUrl).toBe("/login") - expect(deps.reconcileSourcesForWorkspace).not.toHaveBeenCalled() + expect(deps.listSourcesForWorkspace).not.toHaveBeenCalled() }) it("exposes the configured Dashboard origin to the shell", async () => { @@ -93,7 +98,7 @@ describe("loadWorkspaceShellInitialState", () => { const thread = makeThread(workspace.id) const deps = createDependencies({ listChatThreads: vi.fn(async () => [thread]), - reconcileSourcesForWorkspace: vi.fn(async () => [source]), + listSourcesForWorkspace: vi.fn(async () => [source]), sourceViewOptionsBySourceId: vi.fn(() => Effect.succeed(new Map([[source.id, { chunkCount: 2 }]])), ), @@ -119,6 +124,7 @@ describe("loadWorkspaceShellInitialState", () => { chunkCount: 2, }, ]) + expect(deps.ensureDemoChatThread).not.toHaveBeenCalled() }) it("keeps authenticated workspace sources when the demo catalog is unavailable", async () => { @@ -137,7 +143,7 @@ describe("loadWorkspaceShellInitialState", () => { fetchDemoCatalog: vi.fn(async () => { throw new Error("Demo API unavailable.") }), - reconcileSourcesForWorkspace: vi.fn(async () => [legacyFakeSource, source]), + listSourcesForWorkspace: vi.fn(async () => [legacyFakeSource, source]), sourceViewOptionsBySourceId, }) @@ -168,18 +174,22 @@ describe("loadWorkspaceShellInitialState", () => { title: "TSLA-Q4-2025-Update.pdf", knowhereDocumentId: "doc_user_copy", }) + const sourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map())) const deps = createDependencies({ listHiddenDemoSourceIds: vi.fn(async () => ["another-demo"]), - reconcileSourcesForWorkspace: vi.fn(async () => [materializedSource]), + listSourcesForWorkspace: vi.fn(async () => [materializedSource]), + sourceViewOptionsBySourceId, }) const state = await loadWorkspaceShellInitialState(deps) + expect(sourceViewOptionsBySourceId).toHaveBeenCalledWith([], expect.any(Object)) expect(state.sources).toEqual([ expect.objectContaining({ id: "source_demo", kind: "workspace", documentId: "doc_user_copy", + chunkCount: 70, }), ]) }) @@ -195,7 +205,7 @@ describe("loadWorkspaceShellInitialState", () => { }) const sourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map())) const deps = createDependencies({ - reconcileSourcesForWorkspace: vi.fn(async () => [legacyFakeSource]), + listSourcesForWorkspace: vi.fn(async () => [legacyFakeSource]), sourceViewOptionsBySourceId, }) @@ -223,7 +233,7 @@ describe("loadWorkspaceShellInitialState", () => { }) const sourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map())) const deps = createDependencies({ - reconcileSourcesForWorkspace: vi.fn(async () => [nonReadyLegacySource]), + listSourcesForWorkspace: vi.fn(async () => [nonReadyLegacySource]), sourceViewOptionsBySourceId, }) @@ -248,20 +258,77 @@ describe("loadWorkspaceShellInitialState", () => { expect(state.sources).toEqual([]) }) - it("does not seed authenticated empty threads with non-persisted demo chat", async () => { - const state = await loadWorkspaceShellInitialState(createDependencies()) + it("seeds authenticated empty workspaces with persisted demo chat", async () => { + const workspace = makeWorkspace() + const demoThread = makeThread(workspace.id, { + id: "demo_thread_1", + title: "What happened in Tesla Q4?", + demoKey: "knowhere-demo-chat", + }) + const demoMessages = [ + makeMessage(demoThread.id, { + id: "demo_message_user", + role: "user", + content: "What happened in Tesla Q4?", + }), + makeMessage(demoThread.id, { + id: "demo_message_assistant", + role: "assistant", + content: "Tesla delivered higher revenue.", + }), + ] + const ensureDemoChatThread = vi.fn(async () => ({ + thread: demoThread, + messages: demoMessages, + })) + const deps = createDependencies({ + getOptionalAuthenticated: vi.fn(async () => ({ + user: { + id: "user_1", + email: "ada@example.com", + name: "Ada", + }, + workspace, + })), + ensureDemoChatThread, + }) - expect(state.activeChatThreadId).toBeNull() - expect(state.chatMessages).toEqual([]) + const state = await loadWorkspaceShellInitialState(deps) + + expect(ensureDemoChatThread).toHaveBeenCalledWith( + workspace.id, + makeDemoCatalog(), + ) + expect(state.activeChatThreadId).toBe("demo_thread_1") + expect(state.chatThreads).toEqual([ + expect.objectContaining({ + id: "demo_thread_1", + title: "What happened in Tesla Q4?", + }), + ]) + expect(state.chatMessages).toEqual([ + { + id: "demo_message_user", + role: "user", + content: "What happened in Tesla Q4?", + citations: undefined, + }, + { + id: "demo_message_assistant", + role: "assistant", + content: "Tesla delivered higher revenue.", + citations: undefined, + }, + ]) }) - it("reconciles source state during authenticated shell load", async () => { + it("lists workspace sources without blocking on reconciliation", async () => { const workspace = makeWorkspace() const readySource = makeSource(workspace.id, { status: "ready", knowhereDocumentId: "document_1", }) - const reconcileSourcesForWorkspace = vi.fn(async () => [readySource]) + const listSourcesForWorkspace = vi.fn(async () => [readySource]) const deps = { ...createDependencies({ getOptionalAuthenticated: vi.fn(async () => ({ @@ -273,15 +340,12 @@ describe("loadWorkspaceShellInitialState", () => { workspace, })), }), - reconcileSourcesForWorkspace, + listSourcesForWorkspace, } satisfies InitialStateDependencies const state = await loadWorkspaceShellInitialState(deps) - expect(reconcileSourcesForWorkspace).toHaveBeenCalledWith( - workspace, - expect.any(Object), - ) + expect(listSourcesForWorkspace).toHaveBeenCalledWith(workspace.id) expect(state.sources).toEqual([ expect.objectContaining({ id: "demo-tsla-q4-2025", @@ -312,13 +376,14 @@ function createDependencies( return { fetchDemoCatalog: vi.fn(async () => makeDemoCatalog()), - getClientForWorkspace: vi.fn(async () => ({ client })), + getClientForWorkspace: vi.fn(async () => ({ client, apiKey: "sk_test" })), getGuest: vi.fn(async () => ({ loginUrl: "/login" })), getOptionalAuthenticated: vi.fn(async () => ({ user, workspace })), + ensureDemoChatThread: vi.fn(async () => null), listChatThreads: vi.fn(async () => []), listHiddenDemoSourceIds: vi.fn(async () => []), listMessages: vi.fn(async () => []), - reconcileSourcesForWorkspace: vi.fn(async () => []), + listSourcesForWorkspace: vi.fn(async () => []), sourceViewOptionsBySourceId: vi.fn(() => Effect.succeed(new Map())), ...overrides, } @@ -403,7 +468,10 @@ function makeSource( } } -function makeThread(workspaceId: string): ChatThread { +function makeThread( + workspaceId: string, + overrides: Partial = {}, +): ChatThread { return { id: "thread_1", workspaceId, @@ -411,6 +479,23 @@ function makeThread(workspaceId: string): ChatThread { title: "Revenue", createdAt: new Date("2026-05-10T00:00:00.000Z"), updatedAt: new Date("2026-05-10T00:00:00.000Z"), + deletedAt: null, + ...overrides, + } +} + +function makeMessage( + threadId: string, + overrides: Partial = {}, +): ChatMessage { + return { + id: "message_1", + threadId, + role: "user", + content: "Hello", + citations: null, + createdAt: new Date("2026-05-10T00:00:00.000Z"), + ...overrides, } } diff --git a/src/domains/workspace/initial-state.ts b/src/domains/workspace/initial-state.ts index 43604a6..39f2fc5 100644 --- a/src/domains/workspace/initial-state.ts +++ b/src/domains/workspace/initial-state.ts @@ -4,12 +4,18 @@ import { Effect } from "effect" import type { ChatMessageView } from "@/domains/chat/types" import { demoView } from "@/domains/demo/view" -import { resolveWorkspaceDemoSources } from "@/domains/demo/workspace-source-resolution" +import { + getMaterializedDemoSourceViewOptionsBySourceId, + getWorkspaceSourcesNeedingKnowhereChunkCount, + resolveWorkspaceDemoSources, +} from "@/domains/demo/workspace-source-resolution" import { chatThreadService } from "@/domains/chat/thread-service" import { toChatMessageView, toChatThreadView } from "@/domains/chat/view" import { sourceViewOptionsBySourceId as getSourceViewOptionsBySourceId } from "@/domains/sources/counts" -import { reconcileSourcesForWorkspace as reconcileDefaultSourcesForWorkspace } from "@/domains/sources/reconcile" import { sourceService } from "@/domains/sources/service" +import { startBackgroundReconciliation } from "@/domains/sources/background-reconcile" +import { sourceWorkflowRuntime } from "@/domains/sources/workflow-runtime" + import type { SourceView } from "@/domains/sources/types" import { toSourceView } from "@/domains/sources/view" import type { AuthUser } from "@/infrastructure/auth" @@ -42,28 +48,38 @@ type WorkspaceShellInitialState = { } type WorkspaceShellInitialStateClient = - Parameters[1] & - Parameters[1] + Parameters[1] type WorkspaceShellInitialStateDependencies = { readonly fetchDemoCatalog: () => Promise readonly getClientForWorkspace: ( workspace: Workspace, - ) => Promise<{ readonly client: WorkspaceShellInitialStateClient }> + ) => Promise<{ + readonly apiKey: string + readonly client: WorkspaceShellInitialStateClient + }> readonly getGuest: () => Promise<{ readonly loginUrl: string }> readonly getOptionalAuthenticated: () => Promise<{ readonly user: AuthUser readonly workspace: Workspace } | null> - readonly listChatThreads: (workspaceId: string) => Promise + readonly ensureDemoChatThread: ( + workspaceId: string, + catalog: DemoCatalog, + ) => Promise<{ + readonly thread: ChatThread + readonly messages: readonly ChatMessage[] + } | null> + readonly listChatThreads: ( + workspaceId: string, + ) => Promise readonly listHiddenDemoSourceIds: (workspaceId: string) => Promise readonly listMessages: ( workspaceId: string, threadId: string, ) => Promise - readonly reconcileSourcesForWorkspace: ( - workspace: Workspace, - client: WorkspaceShellInitialStateClient, + readonly listSourcesForWorkspace: ( + workspaceId: string, ) => Promise readonly sourceViewOptionsBySourceId: ( sources: readonly Source[], @@ -76,94 +92,155 @@ const defaultDependencies: WorkspaceShellInitialStateDependencies = { getClientForWorkspace: notebookRequestContext.getClientForWorkspace, getGuest: notebookRequestContext.getGuest, getOptionalAuthenticated: notebookRequestContext.getOptionalAuthenticated, + ensureDemoChatThread: chatThreadService.ensureDemo, listChatThreads: chatThreadService.listForWorkspace, listHiddenDemoSourceIds: sourceService.listHiddenDemoSourceIds, listMessages: chatThreadService.listMessages, - reconcileSourcesForWorkspace: reconcileDefaultSourcesForWorkspace, + listSourcesForWorkspace: sourceWorkflowRuntime.listForWorkspace, sourceViewOptionsBySourceId: getSourceViewOptionsBySourceId, } -const emptyDemoCatalog: DemoCatalog = { sources: [] } +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- -export async function loadWorkspaceShellInitialState( +export const loadWorkspaceShellInitialStateEffect = ( deps: WorkspaceShellInitialStateDependencies = defaultDependencies, -): Promise { - const context = await deps.getOptionalAuthenticated() +) => + Effect.gen(function* () { + const context = yield* Effect.tryPromise(() => + deps.getOptionalAuthenticated(), + ) - if (!context) { - const demoCatalog = await deps.fetchDemoCatalog() - const guestContext = await deps.getGuest() - return { - isGuest: true, - sources: demoCatalog.sources.map(demoView.toSourceView), - chatMessages: demoView.toChatMessages(demoCatalog), - dashboardUrl: resolveDashboardUrl(), - loginUrl: guestContext.loginUrl, + if (!context) { + const demoCatalog = yield* Effect.tryPromise(() => + deps.fetchDemoCatalog(), + ) + const guestContext = yield* Effect.tryPromise(() => deps.getGuest()) + return { + isGuest: true, + sources: demoCatalog.sources.map(demoView.toSourceView), + chatMessages: demoView.toChatMessages(demoCatalog), + dashboardUrl: resolveDashboardUrl(), + loginUrl: guestContext.loginUrl, + } } - } - const { user, workspace } = context - const demoCatalog = await fetchOptionalDemoCatalog(deps.fetchDemoCatalog) - const { client } = await deps.getClientForWorkspace(workspace) - const sources = await deps.reconcileSourcesForWorkspace(workspace, client) - const demoSourceResolution = resolveWorkspaceDemoSources( - sources, - demoCatalog, - ) - const hiddenDemoSourceIds = new Set( - await deps.listHiddenDemoSourceIds(workspace.id), - ) - const visibleDemoCatalogSources = demoCatalog.sources - .filter( - (source) => - !demoSourceResolution.materializedDemoSourceIds.has(source.demoSourceId), + const { user, workspace } = context + const demoCatalog = yield* Effect.tryPromise(() => + knowhereDemoApi.fetchOptionalCatalog(deps.fetchDemoCatalog), ) - .filter((source) => !hiddenDemoSourceIds.has(source.demoSourceId)) - const demoSources = visibleDemoCatalogSources.map(demoView.toSourceView) - const chatThreads = await deps.listChatThreads(workspace.id) - const activeChatThread = chatThreads[0] ?? null - const chatMessages = activeChatThread - ? ((await deps.listMessages(workspace.id, activeChatThread.id)) ?? []).map( - (message) => toChatMessageView(message), - ) - : [] - const sourceOptions = await Effect.runPromise( - deps.sourceViewOptionsBySourceId(demoSourceResolution.workspaceSources, client), - ) - - return { - user: { - id: user.id, - name: user.name ?? null, - email: user.email ?? null, - }, - workspace: { - id: workspace.id, - namespace: workspace.namespace, - }, - dashboardUrl: resolveDashboardUrl(), - sources: [ - ...demoSources, - ...demoSourceResolution.workspaceSources.map((source) => - toSourceView(source, sourceOptions.get(source.id)), + const sources = yield* Effect.tryPromise(() => + deps.listSourcesForWorkspace(workspace.id), + ) + const demoSourceResolution = resolveWorkspaceDemoSources( + sources, + demoCatalog, + ) + const hiddenDemoSourceIds = new Set( + yield* Effect.tryPromise(() => + deps.listHiddenDemoSourceIds(workspace.id), ), - ], - chatThreads: chatThreads.map(toChatThreadView), - activeChatThreadId: activeChatThread?.id ?? null, - chatMessages, - } + ) + const visibleDemoCatalogSources = demoCatalog.sources + .filter( + (source) => + !demoSourceResolution.materializedDemoSourceIds.has( + source.demoSourceId, + ), + ) + .filter((source) => !hiddenDemoSourceIds.has(source.demoSourceId)) + const demoSources = visibleDemoCatalogSources.map(demoView.toSourceView) + const listedChatThreads = yield* Effect.tryPromise(() => + deps.listChatThreads(workspace.id), + ) + const seededDemoChatThread = + listedChatThreads.length === 0 + ? yield* Effect.tryPromise(() => + deps.ensureDemoChatThread(workspace.id, demoCatalog), + ) + : null + const chatThreads = seededDemoChatThread + ? [seededDemoChatThread.thread] + : listedChatThreads + const activeChatThread = chatThreads[0] ?? null + const activeChatMessages = seededDemoChatThread + ? seededDemoChatThread.messages + : activeChatThread + ? yield* Effect.tryPromise(() => + deps.listMessages(workspace.id, activeChatThread.id), + ) + : [] + const chatMessages = activeChatMessages + ? activeChatMessages.map((message) => toChatMessageView(message)) + : [] + const sourcesNeedingKnowhereChunkCount = + getWorkspaceSourcesNeedingKnowhereChunkCount( + demoSourceResolution.workspaceSources, + ) + const materializedDemoSourceOptions = + getMaterializedDemoSourceViewOptionsBySourceId( + demoSourceResolution.workspaceSources, + demoCatalog, + ) + const { client, apiKey } = yield* Effect.tryPromise(() => + deps.getClientForWorkspace(workspace), + ) + for (const source of sources) { + if (source.status === "parsing" && source.knowhereJobId) { + yield* Effect.fork( + Effect.tryPromise(() => + startBackgroundReconciliation(workspace.id, source.id, apiKey), + ), + ) + } + } + const sourceOptions = yield* deps.sourceViewOptionsBySourceId( + sourcesNeedingKnowhereChunkCount, + client, + ) + + return { + user: { + id: user.id, + name: user.name ?? null, + email: user.email ?? null, + }, + workspace: { + id: workspace.id, + namespace: workspace.namespace, + }, + dashboardUrl: resolveDashboardUrl(), + sources: [ + ...demoSources, + ...demoSourceResolution.workspaceSources.map((source) => + toSourceView( + source, + materializedDemoSourceOptions.get(source.id) ?? + sourceOptions.get(source.id), + ), + ), + ], + chatThreads: chatThreads.map(toChatThreadView), + activeChatThreadId: activeChatThread?.id ?? null, + chatMessages, + } + }) + +// --------------------------------------------------------------------------- +// Async wrapper (backward-compatible) +// --------------------------------------------------------------------------- + +export async function loadWorkspaceShellInitialState( + deps: WorkspaceShellInitialStateDependencies = defaultDependencies, +): Promise { + return Effect.runPromise(loadWorkspaceShellInitialStateEffect(deps)) } +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + function resolveDashboardUrl(): string | undefined { return process.env.DASHBOARD_ORIGIN } - -async function fetchOptionalDemoCatalog( - fetchDemoCatalog: () => Promise, -): Promise { - try { - return await fetchDemoCatalog() - } catch { - return emptyDemoCatalog - } -} diff --git a/src/domains/workspace/request-context.ts b/src/domains/workspace/request-context.ts index 726502b..dc0e669 100644 --- a/src/domains/workspace/request-context.ts +++ b/src/domains/workspace/request-context.ts @@ -1,10 +1,15 @@ import "server-only" +import { Effect } from "effect" import { headers } from "next/headers" import { ensureApiKeyForWorkspace } from "@/integrations/dashboard/api-key-service" import { authURLs } from "@/infrastructure/auth/urls" -import { getCurrentUser, requireUser, type AuthUser } from "@/infrastructure/auth" +import { + getCurrentUser, + requireUser, + type AuthUser, +} from "@/infrastructure/auth" import { makeKnowhereClient } from "@/integrations/knowhere" import { workspaceService } from "@/domains/workspace/service" import type { Workspace } from "@/infrastructure/db/schema" @@ -25,54 +30,92 @@ type GuestNotebookContext = { readonly loginUrl: string } -async function getAuthenticated(): Promise { - const user = await requireUser() - const workspace = await workspaceService.ensureWorkspace(user.id) +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const getAuthenticatedEffect = Effect.gen(function* () { + const user = yield* Effect.tryPromise(() => requireUser()) + const workspace = yield* Effect.tryPromise(() => + workspaceService.ensureWorkspace(user.id), + ) + + return { user, workspace } + }) + +const getOptionalAuthenticatedEffect = Effect.gen(function* () { + const user = yield* Effect.tryPromise(() => getCurrentUser()) + if (!user) return null + + const workspace = yield* Effect.tryPromise(() => + workspaceService.ensureWorkspace(user.id), + ) + return { user, workspace } + }) + +const getAuthenticatedWithClientEffect = Effect.gen(function* () { + const context = yield* getAuthenticatedEffect + const clientContext = yield* getClientForWorkspaceEffect(context.workspace) + + return { + ...context, + ...clientContext, + } + }) + +const getClientForWorkspaceEffect = (workspace: Workspace) => + Effect.gen(function* () { + const cookieHeader = + (yield* Effect.tryPromise(() => headers())).get("cookie") ?? "" + const apiKey = yield* Effect.tryPromise(() => + ensureApiKeyForWorkspace(workspace.id, cookieHeader), + ) + const client = makeKnowhereClient(apiKey) + + return { apiKey, client } + }) + +const getGuestEffect = Effect.gen(function* () { + const dashboardOrigin = + process.env.DASHBOARD_ORIGIN ?? "http://localhost:3000" + const dashboardLoginURL = `${dashboardOrigin}/login` + const headersList = yield* Effect.tryPromise(() => headers()) + const notebookPublicURL = + process.env.NOTEBOOK_PUBLIC_URL ?? + authURLs.resolveNotebookPublicURLFromHeaders(headersList) + const loginUrl = authURLs.buildDashboardLoginURL( + dashboardLoginURL, + notebookPublicURL, + ) + + return { loginUrl } + }, +) + +// --------------------------------------------------------------------------- +// Async wrappers (backward-compatible) +// --------------------------------------------------------------------------- - return { user, workspace } +async function getAuthenticated(): Promise { + return Effect.runPromise(getAuthenticatedEffect) } async function getOptionalAuthenticated(): Promise { - const user = await getCurrentUser() - if (!user) return null - - const workspace = await workspaceService.ensureWorkspace(user.id) - return { user, workspace } + return Effect.runPromise(getOptionalAuthenticatedEffect) } async function getAuthenticatedWithClient(): Promise { - const context = await getAuthenticated() - const clientContext = await getClientForWorkspace(context.workspace) - - return { - ...context, - ...clientContext, - } + return Effect.runPromise(getAuthenticatedWithClientEffect) } async function getClientForWorkspace( workspace: Workspace, ): Promise> { - const cookieHeader = (await headers()).get("cookie") ?? "" - const apiKey = await ensureApiKeyForWorkspace(workspace.id, cookieHeader) - const client = makeKnowhereClient(apiKey) - - return { apiKey, client } + return Effect.runPromise(getClientForWorkspaceEffect(workspace)) } async function getGuest(): Promise { - const dashboardOrigin = - process.env.DASHBOARD_ORIGIN ?? "http://localhost:3000" - const dashboardLoginURL = `${dashboardOrigin}/login` - const notebookPublicURL = - process.env.NOTEBOOK_PUBLIC_URL ?? - authURLs.resolveNotebookPublicURLFromHeaders(await headers()) - const loginUrl = authURLs.buildDashboardLoginURL( - dashboardLoginURL, - notebookPublicURL, - ) - - return { loginUrl } + return Effect.runPromise(getGuestEffect) } export const notebookRequestContext = { diff --git a/src/infrastructure/auth/index.test.ts b/src/infrastructure/auth/index.test.ts index 67d8168..af43591 100644 --- a/src/infrastructure/auth/index.test.ts +++ b/src/infrastructure/auth/index.test.ts @@ -145,16 +145,20 @@ describe("sessionCookieNames", () => { describe("getCurrentUser", () => { const originalFetch = globalThis.fetch const originalOrigin = process.env.DASHBOARD_ORIGIN + const originalApiKey = process.env.KNOWHERE_API_KEY beforeEach(() => { vi.resetModules() process.env.DASHBOARD_ORIGIN = "https://dashboard.example.test" + delete process.env.KNOWHERE_API_KEY }) afterEach(() => { globalThis.fetch = originalFetch if (originalOrigin === undefined) delete process.env.DASHBOARD_ORIGIN else process.env.DASHBOARD_ORIGIN = originalOrigin + if (originalApiKey === undefined) delete process.env.KNOWHERE_API_KEY + else process.env.KNOWHERE_API_KEY = originalApiKey }) async function loadWithCookie(cookieHeader: string) { @@ -174,6 +178,35 @@ describe("getCurrentUser", () => { expect(fetchSpy).not.toHaveBeenCalled() }) + it("returns the development user when KNOWHERE_API_KEY is configured", async () => { + process.env.KNOWHERE_API_KEY = "sk_dev_key" + delete process.env.DASHBOARD_ORIGIN + const fetchSpy = vi.fn() + globalThis.fetch = fetchSpy + const { getCurrentUser } = await loadWithCookie("") + + const user = await getCurrentUser() + + expect(user).toEqual({ + id: "knowhere-api-key-dev-user", + email: null, + name: "Knowhere API Key Development", + }) + expect(fetchSpy).not.toHaveBeenCalled() + }) + + it("allows requireUser without redirecting when KNOWHERE_API_KEY is configured", async () => { + process.env.KNOWHERE_API_KEY = "sk_dev_key" + delete process.env.DASHBOARD_ORIGIN + const { requireUser } = await loadWithCookie("") + + await expect(requireUser()).resolves.toEqual({ + id: "knowhere-api-key-dev-user", + email: null, + name: "Knowhere API Key Development", + }) + }) + it("POSTs to the Dashboard oRPC endpoint with the incoming Cookie", async () => { const expectedUrl = `https://dashboard.example.test${SESSION_PATH}` const fetchSpy = vi.fn().mockResolvedValue( diff --git a/src/infrastructure/auth/index.ts b/src/infrastructure/auth/index.ts index 7a72c66..95a65b0 100644 --- a/src/infrastructure/auth/index.ts +++ b/src/infrastructure/auth/index.ts @@ -11,6 +11,7 @@ import { import { authURLs } from "./urls" import { sessionCookieNames } from "./session-cookie-names" import { logger } from "@/lib/logger" +import { knowhereApiKeyOverride } from "@/integrations/knowhere-api-key" import { setEmptyJsonBody } from "@/integrations/dashboard/orpc-request" import { formatUnknownForLog } from "@/lib/format-log-value" @@ -57,6 +58,9 @@ const DASHBOARD_SESSION_TIMEOUT_MS = 3_000 // ---- Effect implementation ------------------------------------------------ export const getCurrentUserEffect = Effect.gen(function* () { + const developmentUser = knowhereApiKeyOverride.getDevelopmentUser() + if (developmentUser) return developmentUser + const origin = process.env.DASHBOARD_ORIGIN if (!origin) { return yield* Effect.die( @@ -147,6 +151,14 @@ export const authLayer = Layer.effect( // ---- Public API (Promise-based, for Next.js compatibility) ---------------- export async function getCurrentUser(): Promise { + const developmentUser = knowhereApiKeyOverride.getDevelopmentUser() + if (developmentUser) { + logger.info("auth: using KNOWHERE_API_KEY development user", { + userId: developmentUser.id, + }) + return developmentUser + } + const cookieHeader = (await headers()).get("cookie") ?? "" if (cookieHeader.length === 0) { logger.info("dashboard: POST /api/orpc/users/getCurrentUser skipped (no session cookie)") @@ -202,6 +214,8 @@ export async function requireUser(): Promise { * `getCurrentUser` / `requireUser` before trusting identity. */ export async function hasSessionCookie(): Promise { + if (knowhereApiKeyOverride.hasApiKey()) return true + const jar = await cookies() for (const name of sessionCookieNames()) { if (jar.get(name) !== undefined) return true diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index 4845c44..76f6e2c 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -111,13 +111,9 @@ export const sources = pgTable( deletedAt: timestamp("deleted_at", { withTimezone: true }), }, (t) => [ - // Sidebar list query: per workspace, newest first, soft-deleted - // rows hidden. Partial index keeps the hot path lean. index("sources_workspace_created_idx") .on(t.workspaceId, t.createdAt.desc()) .where(sql`deleted_at IS NULL`), - // Reconcile sweep picks up anything still in `uploading` or - // `parsing`. Small cardinality, small index. index("sources_workspace_status_idx").on(t.workspaceId, t.status), uniqueIndex("sources_workspace_demo_key_idx").on(t.workspaceId, t.demoKey), ], diff --git a/src/integrations/dashboard/api-key-service.test.ts b/src/integrations/dashboard/api-key-service.test.ts index 7ba0b4e..30db09b 100644 --- a/src/integrations/dashboard/api-key-service.test.ts +++ b/src/integrations/dashboard/api-key-service.test.ts @@ -1,6 +1,10 @@ import { afterEach, describe, expect, it, vi } from "vitest" -import { fetchKnowhereJwt, isAuthError } from "./api-key-service" +import { + ensureApiKeyForWorkspace, + fetchKnowhereJwt, + isAuthError, +} from "./api-key-service" function getHeaderValue(headers: HeadersInit | undefined, name: string): string | null { if (headers === undefined) return null @@ -160,3 +164,30 @@ describe("fetchKnowhereJwt", () => { ).rejects.toThrow(/Dashboard JWT issuance: schema mismatch .*"token":""/) }) }) + +describe("ensureApiKeyForWorkspace", () => { + const originalFetch = globalThis.fetch + const originalApiKey = process.env.KNOWHERE_API_KEY + const originalOrigin = process.env.DASHBOARD_ORIGIN + + afterEach(() => { + globalThis.fetch = originalFetch + if (originalApiKey === undefined) delete process.env.KNOWHERE_API_KEY + else process.env.KNOWHERE_API_KEY = originalApiKey + if (originalOrigin === undefined) + delete process.env.DASHBOARD_ORIGIN + else process.env.DASHBOARD_ORIGIN = originalOrigin + }) + + it("uses KNOWHERE_API_KEY without issuing a Dashboard JWT", async () => { + process.env.KNOWHERE_API_KEY = "sk_dev_key" + delete process.env.DASHBOARD_ORIGIN + const fetchSpy = vi.fn() + globalThis.fetch = fetchSpy + + const apiKey = await ensureApiKeyForWorkspace("workspace_1", "") + + expect(apiKey).toBe("sk_dev_key") + expect(fetchSpy).not.toHaveBeenCalled() + }) +}) diff --git a/src/integrations/dashboard/api-key-service.ts b/src/integrations/dashboard/api-key-service.ts index 62ee732..1214cd1 100644 --- a/src/integrations/dashboard/api-key-service.ts +++ b/src/integrations/dashboard/api-key-service.ts @@ -7,6 +7,7 @@ import { HttpClientRequest, } from "@effect/platform" import { logger } from "@/lib/logger" +import { knowhereApiKeyOverride } from "@/integrations/knowhere-api-key" import { setEmptyJsonBody } from "./orpc-request" import { formatUnknownForLog } from "@/lib/format-log-value" @@ -113,13 +114,16 @@ export async function fetchKnowhereJwt( } /** - * Fetch a per-request Knowhere JWT from Dashboard, forwarding the - * incoming session cookie. + * Resolve the credential used for Knowhere SDK calls. Development can + * short-circuit Dashboard JWT issuance by setting KNOWHERE_API_KEY. */ export async function ensureApiKeyForWorkspace( _workspaceId: string, cookieHeader: string, ): Promise { + const apiKey = knowhereApiKeyOverride.getApiKey() + if (apiKey) return apiKey + return fetchKnowhereJwt(cookieHeader) } diff --git a/src/integrations/knowhere-api-key.ts b/src/integrations/knowhere-api-key.ts new file mode 100644 index 0000000..4645c39 --- /dev/null +++ b/src/integrations/knowhere-api-key.ts @@ -0,0 +1,31 @@ +type KnowhereDevelopmentUser = { + readonly id: string + readonly email: string | null + readonly name: string | null +} + +const developmentUser: KnowhereDevelopmentUser = { + id: "knowhere-api-key-dev-user", + email: null, + name: "Knowhere API Key Development", +} + +function getApiKey(): string | null { + const value = process.env.KNOWHERE_API_KEY?.trim() + return value && value.length > 0 ? value : null +} + +function hasApiKey(): boolean { + return getApiKey() !== null +} + +function getDevelopmentUser(): KnowhereDevelopmentUser | null { + if (!hasApiKey()) return null + return developmentUser +} + +export const knowhereApiKeyOverride = { + getApiKey, + hasApiKey, + getDevelopmentUser, +} as const diff --git a/src/integrations/knowhere-demo.test.ts b/src/integrations/knowhere-demo.test.ts index 2b7d917..4018592 100644 --- a/src/integrations/knowhere-demo.test.ts +++ b/src/integrations/knowhere-demo.test.ts @@ -1,12 +1,14 @@ -import { afterEach, describe, expect, it } from "vitest" +import { afterEach, describe, expect, it, vi } from "vitest" import { knowhereDemoApi } from "./knowhere-demo" describe("knowhereDemoApi", () => { const originalBaseURL = process.env.KNOWHERE_BASE_URL + const originalFetch = globalThis.fetch afterEach(() => { restoreEnv("KNOWHERE_BASE_URL", originalBaseURL) + globalThis.fetch = originalFetch }) it("uses the configured Knowhere base URL for demo requests", () => { @@ -24,6 +26,51 @@ describe("knowhereDemoApi", () => { expect(url).toBe("https://api.knowhereto.ai/api/v1/demo/catalog") }) + + it("accepts empty demo chunk content from parser output", async () => { + globalThis.fetch = vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + demo_source_id: "demo-tsla-q4-2025", + canonical_document_id: "demo-doc-tsla-q4-2025", + title: "TSLA-Q4-2025-Update.pdf", + mime_type: "application/pdf", + chunks: [ + { + id: "demo-tsla-q4-2025:chunk-empty", + chunk_id: "chunk-empty", + chunk_type: "text", + content: "", + section_path: "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK", + source_chunk_path: "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK", + file_path: null, + sort_order: 27, + metadata: {}, + asset_url: null, + }, + ], + pagination: { + page: 1, + page_size: 100, + total: 1, + total_pages: 1, + }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ) + + const page = await knowhereDemoApi.fetchChunkPage({ + demoSourceId: "demo-tsla-q4-2025", + page: 1, + pageSize: 100, + }) + + expect(page.chunks[0]).toMatchObject({ + id: "demo-tsla-q4-2025:chunk-empty", + content: "", + }) + }) }) function restoreEnv(key: string, value: string | undefined): void { diff --git a/src/integrations/knowhere-demo.ts b/src/integrations/knowhere-demo.ts index ff8cbd9..37512ca 100644 --- a/src/integrations/knowhere-demo.ts +++ b/src/integrations/knowhere-demo.ts @@ -1,5 +1,7 @@ import "server-only" +import { Effect } from "effect" + export type DemoCitation = { readonly demoSourceId: string readonly canonicalDocumentId: string @@ -175,24 +177,111 @@ type MaterializedDemoSourceResponse = { const DEFAULT_KNOWHERE_BASE_URL = "https://api.knowhereto.ai" -export const knowhereDemoApi = { - fetchCatalog, - fetchChunkPage, - materializeSources, - resolveApiURL, -} as const +const emptyCatalog: DemoCatalog = { sources: [] } -async function fetchCatalog(): Promise { - const response = await fetch(resolveApiURL("/api/v1/demo/catalog"), { - cache: "force-cache", - next: { revalidate: 300 }, - }) - await assertOk(response) +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const fetchCatalogEffect = Effect.fn("knowhereDemo.fetchCatalog")(function* () { + const response = yield* Effect.tryPromise(() => + fetch(resolveApiURL("/api/v1/demo/catalog"), { + cache: "force-cache", + next: { revalidate: 300 }, + }), + ) + yield* assertOkEffect(response) - const body = (await response.json()) as DemoCatalogResponse + const body = (yield* Effect.tryPromise(() => + response.json(), + )) as DemoCatalogResponse return { sources: (body.sources ?? []).map(toDemoSource), } +}) + +const fetchChunkPageEffect = Effect.fn("knowhereDemo.fetchChunkPage")( + function* (input: { + readonly demoSourceId: string + readonly page: number + readonly pageSize: number + }) { + const params = new URLSearchParams({ + page: String(input.page), + page_size: String(input.pageSize), + }) + const response = yield* Effect.tryPromise(() => + fetch( + resolveApiURL( + `/api/v1/demo/sources/${encodeURIComponent(input.demoSourceId)}/chunks?${params.toString()}`, + ), + { cache: "force-cache", next: { revalidate: 300 } }, + ), + ) + yield* assertOkEffect(response) + + return toDemoChunkPage( + (yield* Effect.tryPromise(() => + response.json(), + )) as DemoChunkPageResponse, + ) + }, +) + +const materializeSourcesEffect = Effect.fn("knowhereDemo.materializeSources")( + function* (input: { + readonly apiKey: string + readonly namespace: string + readonly demoSourceIds: readonly string[] + }) { + const requestBody = JSON.stringify({ + namespace: input.namespace, + demo_source_ids: input.demoSourceIds, + }) + const response = yield* Effect.tryPromise(() => + fetch(resolveApiURL("/api/v1/demo/materializations"), { + method: "POST", + headers: { + authorization: `Bearer ${input.apiKey}`, + "content-type": "application/json", + }, + body: requestBody, + }), + ) + yield* assertOkEffect(response) + + const body = (yield* Effect.tryPromise(() => + response.json(), + )) as MaterializeResponse + return (body.sources ?? []).map(toMaterializedDemoSource) + }, +) + +const fetchOptionalCatalogEffect = ( + fetcher?: () => Effect.Effect, +) => + (fetcher ?? fetchCatalogEffect)().pipe( + Effect.catchAll(() => Effect.succeed(emptyCatalog)), + ) + +// --------------------------------------------------------------------------- +// Async wrappers (backward-compatible) +// --------------------------------------------------------------------------- + +async function fetchCatalog(): Promise { + return Effect.runPromise(fetchCatalogEffect()) +} + +async function fetchOptionalCatalog( + fetcher?: () => Promise, +): Promise { + const effectFetcher = fetcher + ? () => + Effect.tryPromise(() => fetcher()).pipe( + Effect.catchAll(() => Effect.succeed(emptyCatalog)), + ) + : undefined + return Effect.runPromise(fetchOptionalCatalogEffect(effectFetcher)) } async function fetchChunkPage(input: { @@ -200,19 +289,7 @@ async function fetchChunkPage(input: { readonly page: number readonly pageSize: number }): Promise { - const params = new URLSearchParams({ - page: String(input.page), - page_size: String(input.pageSize), - }) - const response = await fetch( - resolveApiURL( - `/api/v1/demo/sources/${encodeURIComponent(input.demoSourceId)}/chunks?${params.toString()}`, - ), - { cache: "no-store" }, - ) - await assertOk(response) - - return toDemoChunkPage((await response.json()) as DemoChunkPageResponse) + return Effect.runPromise(fetchChunkPageEffect(input)) } async function materializeSources(input: { @@ -220,35 +297,51 @@ async function materializeSources(input: { readonly namespace: string readonly demoSourceIds: readonly string[] }): Promise { - const response = await fetch(resolveApiURL("/api/v1/demo/materializations"), { - method: "POST", - headers: { - authorization: `Bearer ${input.apiKey}`, - "content-type": "application/json", - }, - body: JSON.stringify({ - namespace: input.namespace, - demo_source_ids: input.demoSourceIds, - }), - }) - await assertOk(response) - - const body = (await response.json()) as MaterializeResponse - return (body.sources ?? []).map(toMaterializedDemoSource) + return Effect.runPromise(materializeSourcesEffect(input)) } +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +export const knowhereDemoApi = { + fetchCatalog, + fetchOptionalCatalog, + fetchChunkPage, + materializeSources, + resolveApiURL, +} as const + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + function resolveApiURL(path: string): string { const baseURL = process.env.KNOWHERE_BASE_URL ?? DEFAULT_KNOWHERE_BASE_URL return new URL(path, baseURL).toString() } -async function assertOk(response: Response): Promise { - if (response.ok) return +class KnowhereDemoApiError { + readonly _tag = "KnowhereDemoApiError" + constructor( + readonly status: number, + readonly body: string, + ) {} +} - const body = await response.text().catch(() => "") - throw new Error( - `Knowhere demo API failed: status=${response.status}, body=${body.slice(0, 300)}`, - ) +function assertOkEffect( + response: Response, +): Effect.Effect { + if (response.ok) return Effect.void + + return Effect.gen(function* () { + const body = yield* Effect.tryPromise(() => + response.text().catch(() => ""), + ).pipe(Effect.orDie) + return yield* Effect.fail( + new KnowhereDemoApiError(response.status, body), + ) + }) } function toDemoSource(source: DemoSourceResponse): DemoSource { @@ -320,7 +413,7 @@ function toDemoChunk( id: requireString(chunk.id), chunkId: requireString(chunk.chunk_id), chunkType: requireString(chunk.chunk_type), - content: requireString(chunk.content), + content: requireContentString(chunk.content), sectionPath: optionalString(chunk.section_path) ?? null, sourceChunkPath: optionalString(chunk.source_chunk_path) ?? null, filePath: optionalString(chunk.file_path) ?? null, @@ -365,6 +458,11 @@ function requireString(value: unknown): string { throw new Error("Expected non-empty string from Knowhere demo API.") } +function requireContentString(value: unknown): string { + if (typeof value === "string") return value + throw new Error("Expected string content from Knowhere demo API.") +} + function optionalString(value: unknown): string | undefined { return typeof value === "string" && value.trim().length > 0 ? value diff --git a/src/lib/api-error-response.ts b/src/lib/api-error-response.ts index 251ddb1..c16a26e 100644 --- a/src/lib/api-error-response.ts +++ b/src/lib/api-error-response.ts @@ -1,5 +1,6 @@ import "server-only" +import { Effect } from "effect" import { NextResponse } from "next/server" import { formatUnknownForLog } from "./format-log-value" @@ -10,13 +11,20 @@ export async function withApiErrorResponse( handler: () => Promise, fallbackMessage: string = "Something went wrong. Please try again.", ): Promise { - try { - return await handler() - } catch (error) { - logger.error("api: unhandled request failure", { - context, - error: formatUnknownForLog(error), - }) - return NextResponse.json({ message: fallbackMessage }, { status: 500 }) - } + return Effect.runPromise( + Effect.tryPromise(handler).pipe( + Effect.catchAll((error) => + Effect.sync(() => { + logger.error("api: unhandled request failure", { + context, + error: formatUnknownForLog(error), + }) + return NextResponse.json( + { message: fallbackMessage }, + { status: 500 }, + ) + }), + ), + ), + ) } diff --git a/src/lib/route-result.ts b/src/lib/route-result.ts index 5b1bb4a..3ec3167 100644 --- a/src/lib/route-result.ts +++ b/src/lib/route-result.ts @@ -1,3 +1,5 @@ +import { Effect } from "effect" + export type RouteResult = { readonly status: number readonly body: TBody @@ -31,22 +33,39 @@ function badRequest(message: string): RouteResult { return error(400, message) } +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const readJsonEffect = ( + request: Request, +): Effect.Effect => + Effect.tryPromise(() => request.json()).pipe( + Effect.map( + (value): ReadJsonResult => ({ ok: true, value }), + ), + Effect.catchAllCause( + (): Effect.Effect => Effect.succeed({ ok: false }), + ), + ) + +const readJsonOrNullEffect = ( + request: Request, +): Effect.Effect => + readJsonEffect(request).pipe( + Effect.map((body) => (body.ok ? body.value : null)), + ) + +// --------------------------------------------------------------------------- +// Async wrappers (backward-compatible) +// --------------------------------------------------------------------------- + async function readJson(request: Request): Promise { - try { - return { - ok: true, - value: await request.json(), - } - } catch { - return { ok: false } - } + return Effect.runPromise(readJsonEffect(request)) } async function readJsonOrNull(request: Request): Promise { - const body = await readJson(request) - if (!body.ok) return null - - return body.value + return Effect.runPromise(readJsonOrNullEffect(request)) } export const routeResult = { diff --git a/src/lib/use-hash-fragment.test.ts b/src/lib/use-hash-fragment.test.ts deleted file mode 100644 index 91ddda4..0000000 --- a/src/lib/use-hash-fragment.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -// @vitest-environment node -import React from "react"; -import { renderToString } from "react-dom/server"; -import { describe, expect, it } from "vitest"; - -import { useHashFragment } from "./use-hash-fragment"; - -describe("useHashFragment", () => { - it("can render on the server without reading window", () => { - function HashFragmentProbe(): React.ReactElement { - const [chunkId] = useHashFragment(); - return React.createElement("span", null, chunkId ?? "none"); - } - - expect(() => renderToString(React.createElement(HashFragmentProbe))).not.toThrow(); - }); -}); diff --git a/src/lib/use-hash-fragment.ts b/src/lib/use-hash-fragment.ts deleted file mode 100644 index 0ab9ddd..0000000 --- a/src/lib/use-hash-fragment.ts +++ /dev/null @@ -1,45 +0,0 @@ -"use client"; - -import { useCallback, useEffect, useState } from "react"; - -const CHUNK_PREFIX = "#chunk-"; - -export function useHashFragment(): [ - chunkId: string | null, - navigateToChunk: (id: string | null) => void, -] { - const [chunkId, setChunkId] = useState(() => - typeof window === "undefined" ? null : readHash(window.location.hash), - ); - - useEffect(() => { - function onHashChange(): void { - setChunkId(readHash(window.location.hash)); - } - onHashChange(); - window.addEventListener("hashchange", onHashChange); - return () => window.removeEventListener("hashchange", onHashChange); - }, []); - - const navigateToChunk = useCallback((id: string | null) => { - if (id) { - window.location.hash = `${CHUNK_PREFIX}${id}`; - } else { - window.history.replaceState( - null, - "", - window.location.pathname + window.location.search, - ); - setChunkId(null); - } - }, []); - - return [chunkId, navigateToChunk]; -} - -function readHash(hash: string): string | null { - if (hash.startsWith(CHUNK_PREFIX)) { - return hash.slice(CHUNK_PREFIX.length); - } - return null; -} diff --git a/src/proxy.test.ts b/src/proxy.test.ts index c88412e..b965699 100644 --- a/src/proxy.test.ts +++ b/src/proxy.test.ts @@ -5,9 +5,11 @@ import { proxy } from "./proxy"; describe("proxy", () => { const originalDashboardOrigin = process.env.DASHBOARD_ORIGIN; + const originalKnowhereApiKey = process.env.KNOWHERE_API_KEY; beforeEach(() => { delete process.env.DASHBOARD_ORIGIN; + delete process.env.KNOWHERE_API_KEY; }); afterEach(() => { @@ -16,6 +18,11 @@ describe("proxy", () => { } else { process.env.DASHBOARD_ORIGIN = originalDashboardOrigin; } + if (originalKnowhereApiKey === undefined) { + delete process.env.KNOWHERE_API_KEY; + } else { + process.env.KNOWHERE_API_KEY = originalKnowhereApiKey; + } }); it("allows anonymous guest source reads", () => { @@ -55,4 +62,16 @@ describe("proxy", () => { "http://localhost:3001/login", ); }); + + it("allows protected app routes without a session when KNOWHERE_API_KEY is configured", () => { + process.env.KNOWHERE_API_KEY = "sk_dev_key"; + + const response = proxy( + new NextRequest("http://localhost:3001/api/sources/source-1", { + method: "PATCH", + }), + ); + + expect(response.headers.get("x-middleware-next")).toBe("1"); + }); }); diff --git a/src/proxy.ts b/src/proxy.ts index ea7a7c0..f245b78 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -1,6 +1,7 @@ import { NextResponse, type NextRequest } from "next/server" import { authURLs } from "@/infrastructure/auth/urls" import { sessionCookieNames } from "@/infrastructure/auth/session-cookie-names" +import { knowhereApiKeyOverride } from "@/integrations/knowhere-api-key" /** * Edge-runtime proxy (renamed from middleware.ts in Next.js 16). @@ -52,6 +53,8 @@ function isGuestSourceReadPath(method: string, pathname: string): boolean { } export function proxy(req: NextRequest): NextResponse { + if (knowhereApiKeyOverride.hasApiKey()) return NextResponse.next() + if (isPublicPath(req)) return NextResponse.next() for (const name of sessionCookieNames()) {