From e021e86d75aabc1d611632cc356e3d108d58ade3 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 01:13:30 +0800 Subject: [PATCH 01/16] feat: support dev api key demo flows --- .env.local.example | 5 + README.md | 7 + .../sources/[sourceId]/chunks/route.test.ts | 125 ++++++++++++++++++ src/domains/chat/chat-thread-repository.ts | 113 +++++++++++++++- src/domains/chat/repository.ts | 2 + src/domains/chat/thread-service.ts | 31 +++++ src/domains/sources/route-chunks.ts | 11 +- .../sources/source-row-repository.test.ts | 54 ++++++++ src/domains/sources/source-row-repository.ts | 15 +++ src/domains/workspace/initial-state.test.ts | 95 ++++++++++++- src/domains/workspace/initial-state.ts | 26 +++- src/infrastructure/auth/index.test.ts | 33 +++++ src/infrastructure/auth/index.ts | 14 ++ .../dashboard/api-key-service.test.ts | 33 ++++- src/integrations/dashboard/api-key-service.ts | 8 +- src/integrations/knowhere-api-key.ts | 31 +++++ src/integrations/knowhere-demo.test.ts | 49 ++++++- src/integrations/knowhere-demo.ts | 7 +- src/proxy.test.ts | 19 +++ src/proxy.ts | 3 + 20 files changed, 665 insertions(+), 16 deletions(-) create mode 100644 src/domains/sources/source-row-repository.test.ts create mode 100644 src/integrations/knowhere-api-key.ts diff --git a/.env.local.example b/.env.local.example index 272d031..f75dd2a 100644 --- a/.env.local.example +++ b/.env.local.example @@ -2,6 +2,11 @@ # Use staging when validating staging keys. # KNOWHERE_BASE_URL=https://api-staging.knowhereto.ai +# Optional development override. When set, Notebook skips Dashboard session +# auth and Dashboard-issued JWT creation, then calls Knowhere directly with +# this key. Leave unset for production and Dashboard-authenticated staging. +# KNOWHERE_API_KEY=sk_your_development_key_here + # --- Chat provider (server-side only) --- # Vercel AI Gateway key; AI SDK picks it up automatically AI_GATEWAY_API_KEY=vck_your_key_here diff --git a/README.md b/README.md index a0111b2..5e4e239 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Upload documents, explore parsed content, and ask questions about your knowledge 2. Fill in your API keys in `.env.local`: - `AI_GATEWAY_API_KEY` — your Vercel AI Gateway key for chat (optional `CHAT_MODEL` override) + - `KNOWHERE_API_KEY` — optional development override that skips Dashboard auth and calls Knowhere directly 3. Install dependencies and run: ```bash @@ -42,6 +43,12 @@ Notebook treats Dashboard as the auth source of truth. Server-side auth calls forward the incoming session cookie to Dashboard oRPC endpoints, including `/api/orpc/users/getCurrentUser` and `/api/orpc/users/issueServiceJwt`. +For local development, setting server-side `KNOWHERE_API_KEY` switches Notebook +into API-key mode. In that mode the app uses a deterministic local development +user, skips Dashboard redirects and JWT issuance, and passes the configured key +directly to the Knowhere SDK. Leave it unset for production and normal +Dashboard-authenticated staging flows. + Dashboard chooses its oRPC handler by request shape and `Content-Type`. When using Effect's `HttpClientRequest.bodyText`, pass `"application/json"` as the body content type. Setting the header before diff --git a/src/app/api/sources/[sourceId]/chunks/route.test.ts b/src/app/api/sources/[sourceId]/chunks/route.test.ts index 4629b41..7eec12f 100644 --- a/src/app/api/sources/[sourceId]/chunks/route.test.ts +++ b/src/app/api/sources/[sourceId]/chunks/route.test.ts @@ -199,6 +199,131 @@ describe("GET /api/sources/[sourceId]/chunks", () => { }) }) + it("serves API-owned demo chunks for authenticated canonical demo sources", async () => { + mocks.getCurrentUser.mockResolvedValue({ + id: "knowhere-api-key-dev-user", + email: null, + name: "Knowhere API Key Development", + }) + mocks.ensureWorkspace.mockResolvedValue({ + id: "workspace_1", + userId: "knowhere-api-key-dev-user", + namespace: "notebook-workspace_1", + createdAt: new Date("2026-05-10T00:00:00.000Z"), + }) + mocks.findSourceInWorkspace.mockResolvedValue(null) + mocks.fetchDemoChunkPage.mockResolvedValue({ + demoSourceId: "demo-tsla-q4-2025", + canonicalDocumentId: "demo-doc-tsla-q4-2025", + title: "TSLA-Q4-2025-Update.pdf", + mimeType: "application/pdf", + chunks: [ + { + id: "demo-tsla-q4-2025:chunk_1", + chunkId: "chunk_1", + chunkType: "text", + content: "Tesla demo content", + sectionPath: "Summary", + sourceChunkPath: "Summary", + filePath: null, + sortOrder: 0, + metadata: {}, + assetUrl: null, + }, + ], + pagination: { + page: 1, + pageSize: 100, + total: 70, + totalPages: 1, + }, + }) + + const response = await GET( + new NextRequest( + "http://localhost:3001/api/sources/demo-tsla-q4-2025/chunks?page=1&pageSize=100", + ), + { params: Promise.resolve({ sourceId: "demo-tsla-q4-2025" }) }, + ) + + await expect(response.json()).resolves.toMatchObject({ + chunks: [ + { + chunkId: "demo-tsla-q4-2025:chunk_1", + documentId: "demo-doc-tsla-q4-2025", + sourceTitle: "TSLA-Q4-2025-Update.pdf", + }, + ], + pagination: { + page: 1, + pageSize: 100, + total: 70, + }, + }) + expect(response.status).toBe(200) + expect(mocks.fetchDemoChunkPage).toHaveBeenCalledWith({ + demoSourceId: "demo-tsla-q4-2025", + page: 1, + pageSize: 100, + }) + expect(mocks.findSourceInWorkspace).toHaveBeenCalledWith( + "workspace_1", + "demo-tsla-q4-2025", + ) + expect(mocks.ensureApiKeyForWorkspace).not.toHaveBeenCalled() + expect(mocks.makeKnowhereClient).not.toHaveBeenCalled() + expect(mocks.getSourceParseAssetUrls).not.toHaveBeenCalled() + }) + + it("logs the demo chunk load failure before returning 404", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined) + try { + mocks.getCurrentUser.mockResolvedValue({ + id: "knowhere-api-key-dev-user", + email: null, + name: "Knowhere API Key Development", + }) + mocks.ensureWorkspace.mockResolvedValue({ + id: "workspace_1", + userId: "knowhere-api-key-dev-user", + namespace: "notebook-workspace_1", + createdAt: new Date("2026-05-10T00:00:00.000Z"), + }) + mocks.findSourceInWorkspace.mockResolvedValue(null) + mocks.fetchDemoChunkPage.mockRejectedValue( + new Error("Knowhere demo API failed: status=404"), + ) + + const response = await GET( + new NextRequest( + "http://localhost:3001/api/sources/demo-tsla-q4-2025/chunks?page=1&pageSize=100", + ), + { params: Promise.resolve({ sourceId: "demo-tsla-q4-2025" }) }, + ) + + expect(response.status).toBe(404) + const line = String(warnSpy.mock.calls[0]?.[0] ?? "") + const log = JSON.parse(line) as { + readonly msg?: unknown + readonly sourceId?: unknown + readonly page?: unknown + readonly pageSize?: unknown + readonly shouldLoadAll?: unknown + readonly error?: unknown + } + expect(log).toMatchObject({ + msg: "sources: demo chunk load failed", + sourceId: "demo-tsla-q4-2025", + page: 1, + pageSize: 100, + shouldLoadAll: false, + error: "Knowhere demo API failed: status=404", + }) + } finally { + warnSpy.mockRestore() + } + }) + it("loads authenticated workspace chunks without probing the demo endpoint first", async () => { const knowhereClient = { documents: { diff --git a/src/domains/chat/chat-thread-repository.ts b/src/domains/chat/chat-thread-repository.ts index 965aee9..7c6f9d1 100644 --- a/src/domains/chat/chat-thread-repository.ts +++ b/src/domains/chat/chat-thread-repository.ts @@ -3,8 +3,32 @@ import "server-only" import { and, desc, eq, isNull, sql } from "drizzle-orm" import { Effect } from "effect" +import { chatCitationPersistence } from "./chat-citation-persistence" import { DbClient } from "@/infrastructure/db" -import { chatThreads, type ChatThread } from "@/infrastructure/db/schema" +import { + chatMessages, + chatThreads, + type ChatMessage, + type ChatThread, +} from "@/infrastructure/db/schema" +import type { ChatCitationView } from "./types" + +type SeedDemoChatMessage = { + readonly role: "user" | "assistant" + readonly content: string + readonly citations?: readonly ChatCitationView[] | null +} + +type SeedDemoChatThreadInput = { + readonly demoKey: string + readonly title: string + readonly messages: readonly SeedDemoChatMessage[] +} + +type SeedDemoChatThreadResult = { + readonly thread: ChatThread + readonly messages: ChatMessage[] +} type ChatThreadRepository = { readonly findThreadInWorkspaceEffect: ( @@ -20,6 +44,10 @@ type ChatThreadRepository = { readonly ensureDefaultThreadEffect: ( workspaceId: string, ) => Effect.Effect + readonly ensureDemoThreadEffect: ( + workspaceId: string, + input: SeedDemoChatThreadInput, + ) => Effect.Effect readonly softDeleteThreadEffect: ( workspaceId: string, threadId: string, @@ -116,6 +144,88 @@ const ensureDefaultThreadEffect: ChatThreadRepository["ensureDefaultThreadEffect return thread }) +const ensureDemoThreadEffect: ChatThreadRepository["ensureDemoThreadEffect"] = + (workspaceId: string, input: SeedDemoChatThreadInput) => + Effect.gen(function* () { + if (input.messages.length === 0) return null + + const db = yield* DbClient + return yield* Effect.promise(() => + db.transaction(async (tx) => { + const insertDemoMessages = async ( + threadId: string, + ): Promise => { + const createdAtMs = Date.now() + return await tx + .insert(chatMessages) + .values( + input.messages.map((message, index) => ({ + threadId, + role: message.role, + content: message.content, + citations: chatCitationPersistence.normalizeCitations( + message.citations, + ), + createdAt: new Date(createdAtMs + index), + })), + ) + .returning() + } + + const existing = ( + await tx + .select() + .from(chatThreads) + .where( + and( + eq(chatThreads.workspaceId, workspaceId), + eq(chatThreads.demoKey, input.demoKey), + ), + ) + .limit(1) + )[0] + + if (existing) { + if (existing.deletedAt !== null) return null + + const existingMessages = await tx + .select() + .from(chatMessages) + .where(eq(chatMessages.threadId, existing.id)) + .orderBy(chatMessages.createdAt) + if (existingMessages.length > 0) { + return { + thread: existing, + messages: existingMessages, + } + } + + const messages = await insertDemoMessages(existing.id) + return { + thread: existing, + messages, + } + } + + const [thread] = await tx + .insert(chatThreads) + .values({ + workspaceId, + demoKey: input.demoKey, + title: input.title, + }) + .returning() + + if (!thread) { + throw new Error("ensureDemoChatThread: insert did not return a row.") + } + + const messages = await insertDemoMessages(thread.id) + return { thread, messages } + }), + ) + }) + const softDeleteThreadEffect: ChatThreadRepository["softDeleteThreadEffect"] = ( workspaceId: string, threadId: string, @@ -144,5 +254,6 @@ export const chatThreadRepository: ChatThreadRepository = { listThreadsForWorkspaceEffect, createThreadEffect, ensureDefaultThreadEffect, + ensureDemoThreadEffect, softDeleteThreadEffect, } diff --git a/src/domains/chat/repository.ts b/src/domains/chat/repository.ts index 1e49080..30ed1a5 100644 --- a/src/domains/chat/repository.ts +++ b/src/domains/chat/repository.ts @@ -8,6 +8,7 @@ type ChatRepository = { readonly listThreadsForWorkspaceEffect: typeof chatThreadRepository.listThreadsForWorkspaceEffect readonly createThreadEffect: typeof chatThreadRepository.createThreadEffect readonly ensureDefaultThreadEffect: typeof chatThreadRepository.ensureDefaultThreadEffect + readonly ensureDemoThreadEffect: typeof chatThreadRepository.ensureDemoThreadEffect readonly listMessagesForThreadEffect: typeof chatMessageRepository.listMessagesForThreadEffect readonly softDeleteThreadEffect: typeof chatThreadRepository.softDeleteThreadEffect readonly appendMessageToThreadEffect: typeof chatMessageRepository.appendMessageToThreadEffect @@ -18,6 +19,7 @@ export const chatRepository: ChatRepository = { listThreadsForWorkspaceEffect: chatThreadRepository.listThreadsForWorkspaceEffect, createThreadEffect: chatThreadRepository.createThreadEffect, ensureDefaultThreadEffect: chatThreadRepository.ensureDefaultThreadEffect, + ensureDemoThreadEffect: chatThreadRepository.ensureDemoThreadEffect, listMessagesForThreadEffect: chatMessageRepository.listMessagesForThreadEffect, softDeleteThreadEffect: chatThreadRepository.softDeleteThreadEffect, appendMessageToThreadEffect: chatMessageRepository.appendMessageToThreadEffect, diff --git a/src/domains/chat/thread-service.ts b/src/domains/chat/thread-service.ts index b5f3275..e22085f 100644 --- a/src/domains/chat/thread-service.ts +++ b/src/domains/chat/thread-service.ts @@ -1,8 +1,10 @@ import "server-only" import { databaseRuntime } from "@/domains/workspace/database-runtime" +import { demoView } from "@/domains/demo/view" import { chatRepository } from "./repository" import type { ChatMessage, ChatThread } from "@/infrastructure/db/schema" +import type { DemoCatalog } from "@/integrations/knowhere-demo" import type { ChatCitationView, CitationView, @@ -18,6 +20,11 @@ type AppendMessageInput = { | null } +type DemoChatThreadSeed = { + readonly thread: ChatThread + readonly messages: ChatMessage[] +} + type ChatThreadService = { readonly findInWorkspace: ( workspaceId: string, @@ -26,6 +33,10 @@ type ChatThreadService = { readonly listForWorkspace: (workspaceId: string) => Promise readonly create: (workspaceId: string) => Promise readonly ensureDefault: (workspaceId: string) => Promise + readonly ensureDemo: ( + workspaceId: string, + catalog: DemoCatalog, + ) => Promise readonly listMessages: ( workspaceId: string, threadId: string, @@ -40,6 +51,8 @@ type ChatThreadService = { ) => Promise } +const seededDemoChatKey = "knowhere-demo-chat" + const findInWorkspace: ChatThreadService["findInWorkspace"] = ( workspaceId: string, threadId: string, @@ -65,6 +78,23 @@ const ensureDefault: ChatThreadService["ensureDefault"] = ( chatRepository.ensureDefaultThreadEffect(workspaceId), ) +const ensureDemo: ChatThreadService["ensureDemo"] = ( + workspaceId: string, + catalog: DemoCatalog, +) => { + const messages = demoView.toChatMessages(catalog) + const firstUserMessage = messages.find((message) => message.role === "user") + if (!firstUserMessage) return Promise.resolve(null) + + return databaseRuntime.runPromise( + chatRepository.ensureDemoThreadEffect(workspaceId, { + demoKey: seededDemoChatKey, + title: firstUserMessage.content, + messages, + }), + ) +} + const listMessages: ChatThreadService["listMessages"] = ( workspaceId: string, threadId: string, @@ -94,6 +124,7 @@ export const chatThreadService: ChatThreadService = { listForWorkspace, create, ensureDefault, + ensureDemo, listMessages, softDelete, appendMessage, diff --git a/src/domains/sources/route-chunks.ts b/src/domains/sources/route-chunks.ts index 17c67dd..8666422 100644 --- a/src/domains/sources/route-chunks.ts +++ b/src/domains/sources/route-chunks.ts @@ -2,6 +2,7 @@ import { Effect } from "effect" import { demoView } from "@/domains/demo/view" import type { DemoChunkPage } from "@/integrations/knowhere-demo" +import { logger } from "@/lib/logger" import { routeResult } from "@/lib/route-result" import { getClientForWorkspace } from "./route-dependencies" import type { @@ -119,7 +120,15 @@ async function loadDemoChunkPage( pagination: page.pagination, }, ) - } catch { + } catch (error) { + logger.warn("sources: demo chunk load failed", { + sourceId: input.sourceId, + page: input.pageParams.page, + pageSize: input.pageParams.pageSize, + shouldLoadAll: input.shouldLoadAll, + knowhereBaseUrl: process.env.KNOWHERE_BASE_URL ?? "(default)", + error: error instanceof Error ? error.message : String(error), + }) return null } } diff --git a/src/domains/sources/source-row-repository.test.ts b/src/domains/sources/source-row-repository.test.ts new file mode 100644 index 0000000..aaa2cce --- /dev/null +++ b/src/domains/sources/source-row-repository.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest" +import { Effect } from "effect" + +import { sourceRowRepository } from "./source-row-repository" +import { DbClient, type Db } from "@/infrastructure/db" + +describe("sourceRowRepository", () => { + it("classifies canonical demo ids as non-workspace source ids", () => { + expect(sourceRowRepository.isWorkspaceSourceId("demo-tsla-q4-2025")).toBe( + false, + ) + }) + + it("classifies UUIDs as workspace source ids", () => { + expect( + sourceRowRepository.isWorkspaceSourceId( + "f03b2dd5-cbc6-44a1-a5cb-8106f8ce52bb", + ), + ).toBe(true) + }) + + it("does not update the database for canonical demo ids", async () => { + const db = makeThrowingDb() + + await expect( + sourceRowRepository.updateInWorkspaceWithDb( + db, + "workspace_1", + "demo-tsla-q4-2025", + { status: "ready" }, + ), + ).resolves.toBeNull() + }) + + it("does not soft-delete the database for canonical demo ids", async () => { + const db = makeThrowingDb() + + await expect( + Effect.runPromise( + sourceRowRepository + .softDeleteEffect("workspace_1", "demo-tsla-q4-2025") + .pipe(Effect.provideService(DbClient, db)), + ), + ).resolves.toBe(false) + }) +}) + +function makeThrowingDb(): Db { + return { + update: () => { + throw new Error("database should not be called for demo source ids") + }, + } as unknown as Db +} diff --git a/src/domains/sources/source-row-repository.ts b/src/domains/sources/source-row-repository.ts index 1ca5178..5f25249 100644 --- a/src/domains/sources/source-row-repository.ts +++ b/src/domains/sources/source-row-repository.ts @@ -68,6 +68,7 @@ type SourceRowRepository = { workspaceId: string, sourceId: string, ) => Effect.Effect + readonly isWorkspaceSourceId: (sourceId: string) => boolean readonly findInWorkspaceWithDb: ( db: Db, workspaceId: string, @@ -187,6 +188,8 @@ const softDeleteEffect: SourceRowRepository["softDeleteEffect"] = ( sourceId: string, ) => Effect.gen(function* () { + if (!isWorkspaceSourceId(sourceId)) return false + const db = yield* DbClient const result = yield* Effect.promise(() => db @@ -222,6 +225,8 @@ async function findInWorkspaceWithDb( workspaceId: string, sourceId: string, ): Promise { + if (!isWorkspaceSourceId(sourceId)) return null + const row = await db .select() .from(sources) @@ -243,6 +248,8 @@ async function updateInWorkspaceWithDb( sourceId: string, values: SourceUpdate, ): Promise { + if (!isWorkspaceSourceId(sourceId)) return null + const [source] = await db .update(sources) .set({ ...values, updatedAt: sql`now()` }) @@ -258,6 +265,13 @@ async function updateInWorkspaceWithDb( return source ?? null } +const WORKSPACE_SOURCE_ID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu + +function isWorkspaceSourceId(sourceId: string): boolean { + return WORKSPACE_SOURCE_ID_PATTERN.test(sourceId) +} + function requireSource(source: Source | null, message: string): Source { if (!source) throw new Error(message) return source @@ -272,6 +286,7 @@ export const sourceRowRepository: SourceRowRepository = { markFailedEffect, clearStagedBlobEffect, softDeleteEffect, + isWorkspaceSourceId, findInWorkspaceWithDb, updateInWorkspaceWithDb, requireSource, diff --git a/src/domains/workspace/initial-state.test.ts b/src/domains/workspace/initial-state.test.ts index fba6767..88cdf5d 100644 --- a/src/domains/workspace/initial-state.test.ts +++ b/src/domains/workspace/initial-state.test.ts @@ -3,7 +3,12 @@ import { afterEach, describe, expect, it, vi } from "vitest" import { loadWorkspaceShellInitialState } from "./initial-state" import type { AuthUser } from "@/infrastructure/auth" -import type { ChatThread, Source, Workspace } from "@/infrastructure/db/schema" +import type { + ChatMessage, + ChatThread, + Source, + Workspace, +} from "@/infrastructure/db/schema" import type { DemoCatalog } from "@/integrations/knowhere-demo" type InitialStateDependencies = NonNullable< @@ -119,6 +124,7 @@ describe("loadWorkspaceShellInitialState", () => { chunkCount: 2, }, ]) + expect(deps.ensureDemoChatThread).not.toHaveBeenCalled() }) it("keeps authenticated workspace sources when the demo catalog is unavailable", async () => { @@ -248,11 +254,68 @@ describe("loadWorkspaceShellInitialState", () => { expect(state.sources).toEqual([]) }) - it("does not seed authenticated empty threads with non-persisted demo chat", async () => { - const state = await loadWorkspaceShellInitialState(createDependencies()) + it("seeds authenticated empty workspaces with persisted demo chat", async () => { + const workspace = makeWorkspace() + const demoThread = makeThread(workspace.id, { + id: "demo_thread_1", + title: "What happened in Tesla Q4?", + demoKey: "knowhere-demo-chat", + }) + const demoMessages = [ + makeMessage(demoThread.id, { + id: "demo_message_user", + role: "user", + content: "What happened in Tesla Q4?", + }), + makeMessage(demoThread.id, { + id: "demo_message_assistant", + role: "assistant", + content: "Tesla delivered higher revenue.", + }), + ] + const ensureDemoChatThread = vi.fn(async () => ({ + thread: demoThread, + messages: demoMessages, + })) + const deps = createDependencies({ + getOptionalAuthenticated: vi.fn(async () => ({ + user: { + id: "user_1", + email: "ada@example.com", + name: "Ada", + }, + workspace, + })), + ensureDemoChatThread, + }) + + const state = await loadWorkspaceShellInitialState(deps) - expect(state.activeChatThreadId).toBeNull() - expect(state.chatMessages).toEqual([]) + expect(ensureDemoChatThread).toHaveBeenCalledWith( + workspace.id, + makeDemoCatalog(), + ) + expect(state.activeChatThreadId).toBe("demo_thread_1") + expect(state.chatThreads).toEqual([ + expect.objectContaining({ + id: "demo_thread_1", + title: "What happened in Tesla Q4?", + }), + ]) + expect(state.chatMessages).toEqual([ + { + id: "demo_message_user", + role: "user", + content: "What happened in Tesla Q4?", + citations: undefined, + }, + { + id: "demo_message_assistant", + role: "assistant", + content: "Tesla delivered higher revenue.", + citations: undefined, + }, + ]) }) it("reconciles source state during authenticated shell load", async () => { @@ -315,6 +378,7 @@ function createDependencies( getClientForWorkspace: vi.fn(async () => ({ client })), getGuest: vi.fn(async () => ({ loginUrl: "/login" })), getOptionalAuthenticated: vi.fn(async () => ({ user, workspace })), + ensureDemoChatThread: vi.fn(async () => null), listChatThreads: vi.fn(async () => []), listHiddenDemoSourceIds: vi.fn(async () => []), listMessages: vi.fn(async () => []), @@ -403,7 +467,10 @@ function makeSource( } } -function makeThread(workspaceId: string): ChatThread { +function makeThread( + workspaceId: string, + overrides: Partial = {}, +): ChatThread { return { id: "thread_1", workspaceId, @@ -412,5 +479,21 @@ function makeThread(workspaceId: string): ChatThread { createdAt: new Date("2026-05-10T00:00:00.000Z"), updatedAt: new Date("2026-05-10T00:00:00.000Z"), deletedAt: null, + ...overrides, + } +} + +function makeMessage( + threadId: string, + overrides: Partial = {}, +): ChatMessage { + return { + id: "message_1", + threadId, + role: "user", + content: "Hello", + citations: null, + createdAt: new Date("2026-05-10T00:00:00.000Z"), + ...overrides, } } diff --git a/src/domains/workspace/initial-state.ts b/src/domains/workspace/initial-state.ts index 43604a6..e1a5e82 100644 --- a/src/domains/workspace/initial-state.ts +++ b/src/domains/workspace/initial-state.ts @@ -55,6 +55,13 @@ type WorkspaceShellInitialStateDependencies = { readonly user: AuthUser readonly workspace: Workspace } | null> + readonly ensureDemoChatThread: ( + workspaceId: string, + catalog: DemoCatalog, + ) => Promise<{ + readonly thread: ChatThread + readonly messages: readonly ChatMessage[] + } | null> readonly listChatThreads: (workspaceId: string) => Promise readonly listHiddenDemoSourceIds: (workspaceId: string) => Promise readonly listMessages: ( @@ -76,6 +83,7 @@ const defaultDependencies: WorkspaceShellInitialStateDependencies = { getClientForWorkspace: notebookRequestContext.getClientForWorkspace, getGuest: notebookRequestContext.getGuest, getOptionalAuthenticated: notebookRequestContext.getOptionalAuthenticated, + ensureDemoChatThread: chatThreadService.ensureDemo, listChatThreads: chatThreadService.listForWorkspace, listHiddenDemoSourceIds: sourceService.listHiddenDemoSourceIds, listMessages: chatThreadService.listMessages, @@ -120,10 +128,22 @@ export async function loadWorkspaceShellInitialState( ) .filter((source) => !hiddenDemoSourceIds.has(source.demoSourceId)) const demoSources = visibleDemoCatalogSources.map(demoView.toSourceView) - const chatThreads = await deps.listChatThreads(workspace.id) + const listedChatThreads = await deps.listChatThreads(workspace.id) + const seededDemoChatThread = + listedChatThreads.length === 0 + ? await deps.ensureDemoChatThread(workspace.id, demoCatalog) + : null + const chatThreads = seededDemoChatThread + ? [seededDemoChatThread.thread] + : listedChatThreads const activeChatThread = chatThreads[0] ?? null - const chatMessages = activeChatThread - ? ((await deps.listMessages(workspace.id, activeChatThread.id)) ?? []).map( + const activeChatMessages = seededDemoChatThread + ? seededDemoChatThread.messages + : activeChatThread + ? await deps.listMessages(workspace.id, activeChatThread.id) + : [] + const chatMessages = activeChatMessages + ? activeChatMessages.map( (message) => toChatMessageView(message), ) : [] diff --git a/src/infrastructure/auth/index.test.ts b/src/infrastructure/auth/index.test.ts index 67d8168..af43591 100644 --- a/src/infrastructure/auth/index.test.ts +++ b/src/infrastructure/auth/index.test.ts @@ -145,16 +145,20 @@ describe("sessionCookieNames", () => { describe("getCurrentUser", () => { const originalFetch = globalThis.fetch const originalOrigin = process.env.DASHBOARD_ORIGIN + const originalApiKey = process.env.KNOWHERE_API_KEY beforeEach(() => { vi.resetModules() process.env.DASHBOARD_ORIGIN = "https://dashboard.example.test" + delete process.env.KNOWHERE_API_KEY }) afterEach(() => { globalThis.fetch = originalFetch if (originalOrigin === undefined) delete process.env.DASHBOARD_ORIGIN else process.env.DASHBOARD_ORIGIN = originalOrigin + if (originalApiKey === undefined) delete process.env.KNOWHERE_API_KEY + else process.env.KNOWHERE_API_KEY = originalApiKey }) async function loadWithCookie(cookieHeader: string) { @@ -174,6 +178,35 @@ describe("getCurrentUser", () => { expect(fetchSpy).not.toHaveBeenCalled() }) + it("returns the development user when KNOWHERE_API_KEY is configured", async () => { + process.env.KNOWHERE_API_KEY = "sk_dev_key" + delete process.env.DASHBOARD_ORIGIN + const fetchSpy = vi.fn() + globalThis.fetch = fetchSpy + const { getCurrentUser } = await loadWithCookie("") + + const user = await getCurrentUser() + + expect(user).toEqual({ + id: "knowhere-api-key-dev-user", + email: null, + name: "Knowhere API Key Development", + }) + expect(fetchSpy).not.toHaveBeenCalled() + }) + + it("allows requireUser without redirecting when KNOWHERE_API_KEY is configured", async () => { + process.env.KNOWHERE_API_KEY = "sk_dev_key" + delete process.env.DASHBOARD_ORIGIN + const { requireUser } = await loadWithCookie("") + + await expect(requireUser()).resolves.toEqual({ + id: "knowhere-api-key-dev-user", + email: null, + name: "Knowhere API Key Development", + }) + }) + it("POSTs to the Dashboard oRPC endpoint with the incoming Cookie", async () => { const expectedUrl = `https://dashboard.example.test${SESSION_PATH}` const fetchSpy = vi.fn().mockResolvedValue( diff --git a/src/infrastructure/auth/index.ts b/src/infrastructure/auth/index.ts index 7a72c66..95a65b0 100644 --- a/src/infrastructure/auth/index.ts +++ b/src/infrastructure/auth/index.ts @@ -11,6 +11,7 @@ import { import { authURLs } from "./urls" import { sessionCookieNames } from "./session-cookie-names" import { logger } from "@/lib/logger" +import { knowhereApiKeyOverride } from "@/integrations/knowhere-api-key" import { setEmptyJsonBody } from "@/integrations/dashboard/orpc-request" import { formatUnknownForLog } from "@/lib/format-log-value" @@ -57,6 +58,9 @@ const DASHBOARD_SESSION_TIMEOUT_MS = 3_000 // ---- Effect implementation ------------------------------------------------ export const getCurrentUserEffect = Effect.gen(function* () { + const developmentUser = knowhereApiKeyOverride.getDevelopmentUser() + if (developmentUser) return developmentUser + const origin = process.env.DASHBOARD_ORIGIN if (!origin) { return yield* Effect.die( @@ -147,6 +151,14 @@ export const authLayer = Layer.effect( // ---- Public API (Promise-based, for Next.js compatibility) ---------------- export async function getCurrentUser(): Promise { + const developmentUser = knowhereApiKeyOverride.getDevelopmentUser() + if (developmentUser) { + logger.info("auth: using KNOWHERE_API_KEY development user", { + userId: developmentUser.id, + }) + return developmentUser + } + const cookieHeader = (await headers()).get("cookie") ?? "" if (cookieHeader.length === 0) { logger.info("dashboard: POST /api/orpc/users/getCurrentUser skipped (no session cookie)") @@ -202,6 +214,8 @@ export async function requireUser(): Promise { * `getCurrentUser` / `requireUser` before trusting identity. */ export async function hasSessionCookie(): Promise { + if (knowhereApiKeyOverride.hasApiKey()) return true + const jar = await cookies() for (const name of sessionCookieNames()) { if (jar.get(name) !== undefined) return true diff --git a/src/integrations/dashboard/api-key-service.test.ts b/src/integrations/dashboard/api-key-service.test.ts index 7ba0b4e..30db09b 100644 --- a/src/integrations/dashboard/api-key-service.test.ts +++ b/src/integrations/dashboard/api-key-service.test.ts @@ -1,6 +1,10 @@ import { afterEach, describe, expect, it, vi } from "vitest" -import { fetchKnowhereJwt, isAuthError } from "./api-key-service" +import { + ensureApiKeyForWorkspace, + fetchKnowhereJwt, + isAuthError, +} from "./api-key-service" function getHeaderValue(headers: HeadersInit | undefined, name: string): string | null { if (headers === undefined) return null @@ -160,3 +164,30 @@ describe("fetchKnowhereJwt", () => { ).rejects.toThrow(/Dashboard JWT issuance: schema mismatch .*"token":""/) }) }) + +describe("ensureApiKeyForWorkspace", () => { + const originalFetch = globalThis.fetch + const originalApiKey = process.env.KNOWHERE_API_KEY + const originalOrigin = process.env.DASHBOARD_ORIGIN + + afterEach(() => { + globalThis.fetch = originalFetch + if (originalApiKey === undefined) delete process.env.KNOWHERE_API_KEY + else process.env.KNOWHERE_API_KEY = originalApiKey + if (originalOrigin === undefined) + delete process.env.DASHBOARD_ORIGIN + else process.env.DASHBOARD_ORIGIN = originalOrigin + }) + + it("uses KNOWHERE_API_KEY without issuing a Dashboard JWT", async () => { + process.env.KNOWHERE_API_KEY = "sk_dev_key" + delete process.env.DASHBOARD_ORIGIN + const fetchSpy = vi.fn() + globalThis.fetch = fetchSpy + + const apiKey = await ensureApiKeyForWorkspace("workspace_1", "") + + expect(apiKey).toBe("sk_dev_key") + expect(fetchSpy).not.toHaveBeenCalled() + }) +}) diff --git a/src/integrations/dashboard/api-key-service.ts b/src/integrations/dashboard/api-key-service.ts index 62ee732..1214cd1 100644 --- a/src/integrations/dashboard/api-key-service.ts +++ b/src/integrations/dashboard/api-key-service.ts @@ -7,6 +7,7 @@ import { HttpClientRequest, } from "@effect/platform" import { logger } from "@/lib/logger" +import { knowhereApiKeyOverride } from "@/integrations/knowhere-api-key" import { setEmptyJsonBody } from "./orpc-request" import { formatUnknownForLog } from "@/lib/format-log-value" @@ -113,13 +114,16 @@ export async function fetchKnowhereJwt( } /** - * Fetch a per-request Knowhere JWT from Dashboard, forwarding the - * incoming session cookie. + * Resolve the credential used for Knowhere SDK calls. Development can + * short-circuit Dashboard JWT issuance by setting KNOWHERE_API_KEY. */ export async function ensureApiKeyForWorkspace( _workspaceId: string, cookieHeader: string, ): Promise { + const apiKey = knowhereApiKeyOverride.getApiKey() + if (apiKey) return apiKey + return fetchKnowhereJwt(cookieHeader) } diff --git a/src/integrations/knowhere-api-key.ts b/src/integrations/knowhere-api-key.ts new file mode 100644 index 0000000..4645c39 --- /dev/null +++ b/src/integrations/knowhere-api-key.ts @@ -0,0 +1,31 @@ +type KnowhereDevelopmentUser = { + readonly id: string + readonly email: string | null + readonly name: string | null +} + +const developmentUser: KnowhereDevelopmentUser = { + id: "knowhere-api-key-dev-user", + email: null, + name: "Knowhere API Key Development", +} + +function getApiKey(): string | null { + const value = process.env.KNOWHERE_API_KEY?.trim() + return value && value.length > 0 ? value : null +} + +function hasApiKey(): boolean { + return getApiKey() !== null +} + +function getDevelopmentUser(): KnowhereDevelopmentUser | null { + if (!hasApiKey()) return null + return developmentUser +} + +export const knowhereApiKeyOverride = { + getApiKey, + hasApiKey, + getDevelopmentUser, +} as const diff --git a/src/integrations/knowhere-demo.test.ts b/src/integrations/knowhere-demo.test.ts index 2b7d917..4018592 100644 --- a/src/integrations/knowhere-demo.test.ts +++ b/src/integrations/knowhere-demo.test.ts @@ -1,12 +1,14 @@ -import { afterEach, describe, expect, it } from "vitest" +import { afterEach, describe, expect, it, vi } from "vitest" import { knowhereDemoApi } from "./knowhere-demo" describe("knowhereDemoApi", () => { const originalBaseURL = process.env.KNOWHERE_BASE_URL + const originalFetch = globalThis.fetch afterEach(() => { restoreEnv("KNOWHERE_BASE_URL", originalBaseURL) + globalThis.fetch = originalFetch }) it("uses the configured Knowhere base URL for demo requests", () => { @@ -24,6 +26,51 @@ describe("knowhereDemoApi", () => { expect(url).toBe("https://api.knowhereto.ai/api/v1/demo/catalog") }) + + it("accepts empty demo chunk content from parser output", async () => { + globalThis.fetch = vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + demo_source_id: "demo-tsla-q4-2025", + canonical_document_id: "demo-doc-tsla-q4-2025", + title: "TSLA-Q4-2025-Update.pdf", + mime_type: "application/pdf", + chunks: [ + { + id: "demo-tsla-q4-2025:chunk-empty", + chunk_id: "chunk-empty", + chunk_type: "text", + content: "", + section_path: "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK", + source_chunk_path: "Default_Root/TSLA-Q4-2025-Update.pdf-->OUTLOOK", + file_path: null, + sort_order: 27, + metadata: {}, + asset_url: null, + }, + ], + pagination: { + page: 1, + page_size: 100, + total: 1, + total_pages: 1, + }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ) + + const page = await knowhereDemoApi.fetchChunkPage({ + demoSourceId: "demo-tsla-q4-2025", + page: 1, + pageSize: 100, + }) + + expect(page.chunks[0]).toMatchObject({ + id: "demo-tsla-q4-2025:chunk-empty", + content: "", + }) + }) }) function restoreEnv(key: string, value: string | undefined): void { diff --git a/src/integrations/knowhere-demo.ts b/src/integrations/knowhere-demo.ts index ff8cbd9..2f5a7dd 100644 --- a/src/integrations/knowhere-demo.ts +++ b/src/integrations/knowhere-demo.ts @@ -320,7 +320,7 @@ function toDemoChunk( id: requireString(chunk.id), chunkId: requireString(chunk.chunk_id), chunkType: requireString(chunk.chunk_type), - content: requireString(chunk.content), + content: requireContentString(chunk.content), sectionPath: optionalString(chunk.section_path) ?? null, sourceChunkPath: optionalString(chunk.source_chunk_path) ?? null, filePath: optionalString(chunk.file_path) ?? null, @@ -365,6 +365,11 @@ function requireString(value: unknown): string { throw new Error("Expected non-empty string from Knowhere demo API.") } +function requireContentString(value: unknown): string { + if (typeof value === "string") return value + throw new Error("Expected string content from Knowhere demo API.") +} + function optionalString(value: unknown): string | undefined { return typeof value === "string" && value.trim().length > 0 ? value diff --git a/src/proxy.test.ts b/src/proxy.test.ts index c88412e..b965699 100644 --- a/src/proxy.test.ts +++ b/src/proxy.test.ts @@ -5,9 +5,11 @@ import { proxy } from "./proxy"; describe("proxy", () => { const originalDashboardOrigin = process.env.DASHBOARD_ORIGIN; + const originalKnowhereApiKey = process.env.KNOWHERE_API_KEY; beforeEach(() => { delete process.env.DASHBOARD_ORIGIN; + delete process.env.KNOWHERE_API_KEY; }); afterEach(() => { @@ -16,6 +18,11 @@ describe("proxy", () => { } else { process.env.DASHBOARD_ORIGIN = originalDashboardOrigin; } + if (originalKnowhereApiKey === undefined) { + delete process.env.KNOWHERE_API_KEY; + } else { + process.env.KNOWHERE_API_KEY = originalKnowhereApiKey; + } }); it("allows anonymous guest source reads", () => { @@ -55,4 +62,16 @@ describe("proxy", () => { "http://localhost:3001/login", ); }); + + it("allows protected app routes without a session when KNOWHERE_API_KEY is configured", () => { + process.env.KNOWHERE_API_KEY = "sk_dev_key"; + + const response = proxy( + new NextRequest("http://localhost:3001/api/sources/source-1", { + method: "PATCH", + }), + ); + + expect(response.headers.get("x-middleware-next")).toBe("1"); + }); }); diff --git a/src/proxy.ts b/src/proxy.ts index ea7a7c0..f245b78 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -1,6 +1,7 @@ import { NextResponse, type NextRequest } from "next/server" import { authURLs } from "@/infrastructure/auth/urls" import { sessionCookieNames } from "@/infrastructure/auth/session-cookie-names" +import { knowhereApiKeyOverride } from "@/integrations/knowhere-api-key" /** * Edge-runtime proxy (renamed from middleware.ts in Next.js 16). @@ -52,6 +53,8 @@ function isGuestSourceReadPath(method: string, pathname: string): boolean { } export function proxy(req: NextRequest): NextResponse { + if (knowhereApiKeyOverride.hasApiKey()) return NextResponse.next() + if (isPublicPath(req)) return NextResponse.next() for (const name of sessionCookieNames()) { From ca60725921083224b1c6fd2c3d3b7e02214baa7e Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 01:23:15 +0800 Subject: [PATCH 02/16] fix: make grounded answers friendlier --- src/domains/chat/index.test.ts | 17 +++++++++++++++++ src/domains/chat/prompt.ts | 5 +++++ 2 files changed, 22 insertions(+) diff --git a/src/domains/chat/index.test.ts b/src/domains/chat/index.test.ts index 90627e7..835fabf 100644 --- a/src/domains/chat/index.test.ts +++ b/src/domains/chat/index.test.ts @@ -340,6 +340,23 @@ describe("buildGroundedPrompt", () => { expect(prompt).toContain("requirements.txt"); expect(prompt).toContain("don't answer the question, say so directly"); }); + + it("asks the model to answer naturally and directly", () => { + const prompt = buildGroundedPrompt({ + question: "How about the TBD?", + results: [ + makeRetrievalResult({ + content: "Roadster location: TBD. Status: Design development.", + }), + ], + }); + + expect(prompt).toContain("Answer in a natural, friendly, and direct tone."); + expect(prompt).toContain("Start with the answer first."); + expect(prompt).toContain("Avoid meta phrases like \"Based on the sources\""); + expect(prompt).toContain("Keep answers concise by default"); + expect(prompt).toContain("I don't see more detail in these sources"); + }); }); describe("buildRetrievalQueryPrompt", () => { diff --git a/src/domains/chat/prompt.ts b/src/domains/chat/prompt.ts index 0620306..ab715e4 100644 --- a/src/domains/chat/prompt.ts +++ b/src/domains/chat/prompt.ts @@ -144,6 +144,11 @@ export function buildGroundedPrompt(input: BuildGroundedPromptInput): string { "You are an assistant that answers questions from provided source excerpts.", "Your answer must be grounded only in the sources below. If they don't answer the question, say so directly.", "Use the recent conversation only to resolve references like \"this document\"; do not use it as factual evidence.", + "Answer in a natural, friendly, and direct tone.", + "Start with the answer first. Avoid meta phrases like \"Based on the sources\" or \"Based on the source excerpts\" unless the user asks how you know.", + "Use plain language. Prefer \"I don't see more detail in these sources\" over formal wording like \"the sources do not specify\".", + "Keep answers concise by default: 1-3 short paragraphs unless the user asks for detail.", + "Do not over-explain uncertainty. State what is known, then briefly state what is not shown in the sources.", "CITATION FORMAT: After each sourced statement include a brief citation label like [Source N: what the source says]. Use only the provided source numbers.", "", `Question: ${input.question}`, From 4450cee972c53287d09ae7ca68d8505b90f882f8 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 01:55:56 +0800 Subject: [PATCH 03/16] fix: keep chunk panel focused --- src/components/chunks-panel-workflow.ts | 39 ++++++++++--- src/components/chunks-panel.test.ts | 50 ++++++++++++++++ src/components/source-panel-state.test.ts | 6 +- src/components/source-panel-state.ts | 8 +-- src/components/sources-panel.tsx | 1 - src/components/workspace-shell.test.ts | 58 +++++++++++++++++++ src/components/workspace-source-state.test.ts | 44 +++++++++++++- src/components/workspace-source-state.ts | 35 +++++++++-- .../workspace-source-workflow.test.ts | 2 +- src/components/workspace-source-workflow.ts | 10 +++- 10 files changed, 224 insertions(+), 29 deletions(-) diff --git a/src/components/chunks-panel-workflow.ts b/src/components/chunks-panel-workflow.ts index 7439077..a0e9a99 100644 --- a/src/components/chunks-panel-workflow.ts +++ b/src/components/chunks-panel-workflow.ts @@ -50,6 +50,12 @@ type ChunksPanelWorkflow = { readonly visibleView: ChunksPanelView } +type LocalFocusedChunk = { + readonly chunkId: string + readonly parentRequestId: number + readonly requestId: number +} + const estimatedChunkCardHeight = 220 const virtualListOverscan = 4 const infiniteScrollThreshold = 720 @@ -67,9 +73,8 @@ export function useChunksPanelWorkflow({ }: ChunksPanelWorkflowInput): ChunksPanelWorkflow { const viewportRef = useRef(null) const [activeView, setActiveView] = useState("parsed") - const [localFocusedChunkId, setLocalFocusedChunkId] = useState( - null, - ) + const [localFocusedChunk, setLocalFocusedChunk] = + useState(null) const [originalTargetPage, setOriginalTargetPage] = useState<{ readonly pageNumber: number | null readonly requestId: number @@ -77,7 +82,14 @@ export function useChunksPanelWorkflow({ pageNumber: null, requestId: 0, }) - const activeFocusedChunkId = focusedChunkId ?? localFocusedChunkId + const activeFocusedChunkId: string | null = + localFocusedChunk?.parentRequestId === focusedChunkRequestId + ? localFocusedChunk.chunkId + : focusedChunkId + const activeFocusedChunkRequestId: number = + localFocusedChunk?.parentRequestId === focusedChunkRequestId + ? localFocusedChunk.requestId + : focusedChunkRequestId const hasOriginalFile = selectedSource !== null && selectedSourceFile !== null const visibleView = hasOriginalFile ? activeView : "parsed" const visibleChunks = useMemo( @@ -157,8 +169,17 @@ export function useChunksPanelWorkflow({ ) const requestChunkFocus = useCallback((chunkId: string | null): void => { - setLocalFocusedChunkId(chunkId) - }, []) + if (chunkId === null) { + setLocalFocusedChunk(null) + return + } + + setLocalFocusedChunk((current: LocalFocusedChunk | null) => ({ + chunkId, + parentRequestId: focusedChunkRequestId, + requestId: (current?.requestId ?? 0) + 1, + })) + }, [focusedChunkRequestId]) const handleChunkSelected = useCallback( (chunk: ParsedChunkView): void => { @@ -198,7 +219,7 @@ export function useChunksPanelWorkflow({ } scrollToFocusedChunk() - }, [activeFocusedChunkId, focusedChunkRequestId, scrollToFocusedChunk]) + }, [activeFocusedChunkId, activeFocusedChunkRequestId, scrollToFocusedChunk]) useEffect(() => { if (!hasOriginalFile) setActiveView("parsed") @@ -208,6 +229,10 @@ export function useChunksPanelWorkflow({ if (focusedChunkId) setActiveView("parsed") }, [focusedChunkId, focusedChunkRequestId]) + useEffect(() => { + setLocalFocusedChunk(null) + }, [selectedSource]) + return { activeFocusedChunkId, handleChunkSelected, diff --git a/src/components/chunks-panel.test.ts b/src/components/chunks-panel.test.ts index 29e55b6..281616f 100644 --- a/src/components/chunks-panel.test.ts +++ b/src/components/chunks-panel.test.ts @@ -623,6 +623,56 @@ describe("ChunksPanel", () => { ).toBe("true"); }); + it("lets in-chunk table references override the current citation focus", async () => { + mockVisibleVirtualViewport(); + const user = userEvent.setup(); + + render( + React.createElement(C, { + chunks: [ + { + chunkId: "text_1", + parserChunkId: "parser_text_1", + type: "text", + content: "See [tables/table-1.html] for Roadster details.", + sourceTitle: "manual.pdf", + connections: [ + { + targetParserChunkId: "parser_table_1", + targetChunkId: "table_1", + relation: "embeds", + ref: "[tables/table-1.html]", + position: { start: 4, end: 25 }, + }, + ], + }, + { + chunkId: "table_1", + parserChunkId: "parser_table_1", + type: "table", + content: + "
RoadsterTBD
", + sourceTitle: "manual.pdf", + }, + ], + selectedSource: "manual.pdf", + focusedChunkId: "text_1", + focusedChunkRequestId: 1, + }), + ); + + await user.click(screen.getByRole("button", { name: "Table 1" })); + + await waitFor(() => { + const focusedRow = screen + .getByTestId("chunk-card-shell-table_1") + .closest("[data-index]"); + + expect(focusedRow?.getAttribute("data-index")).toBe("0"); + expect(focusedRow?.getAttribute("data-focused-chunk")).toBe("true"); + }); + }); + it("renders a focused virtual chunk outside the initial range first", async () => { mockVisibleVirtualViewport(); diff --git a/src/components/source-panel-state.test.ts b/src/components/source-panel-state.test.ts index 5bd7eb9..e7d83b8 100644 --- a/src/components/source-panel-state.test.ts +++ b/src/components/source-panel-state.test.ts @@ -23,18 +23,16 @@ describe("sourcePanelState", () => { expect(state.archivingSourceIdSet.has("source_1")).toBe(true); }); - it("selects or clears the current Source from a row click", () => { + it("selects a Source from a row click without clearing the current selection", () => { expect( sourcePanelState.getNextSelectedSourceId({ sourceId: "source_1", - selectedSourceId: null, }), ).toBe("source_1"); expect( sourcePanelState.getNextSelectedSourceId({ sourceId: "source_1", - selectedSourceId: "source_1", }), - ).toBeNull(); + ).toBe("source_1"); }); }); diff --git a/src/components/source-panel-state.ts b/src/components/source-panel-state.ts index 2f88600..07628a8 100644 --- a/src/components/source-panel-state.ts +++ b/src/components/source-panel-state.ts @@ -13,7 +13,6 @@ type ArchiveConfirmationState = { } type NextSelectedSourceInput = { - readonly selectedSourceId: string | null readonly sourceId: string } @@ -48,11 +47,8 @@ function getArchiveConfirmationState({ } } -function getNextSelectedSourceId({ - selectedSourceId, - sourceId, -}: NextSelectedSourceInput): string | null { - return sourceId === selectedSourceId ? null : sourceId +function getNextSelectedSourceId({ sourceId }: NextSelectedSourceInput): string | null { + return sourceId } function shouldCloseArchiveConfirmation( diff --git a/src/components/sources-panel.tsx b/src/components/sources-panel.tsx index 0f96003..e947c49 100644 --- a/src/components/sources-panel.tsx +++ b/src/components/sources-panel.tsx @@ -137,7 +137,6 @@ export function SourcesPanel({ onSelectSource?.( sourcePanelState.getNextSelectedSourceId({ sourceId: source.id, - selectedSourceId, }), ) } diff --git a/src/components/workspace-shell.test.ts b/src/components/workspace-shell.test.ts index ea5255a..a4e1909 100644 --- a/src/components/workspace-shell.test.ts +++ b/src/components/workspace-shell.test.ts @@ -159,6 +159,64 @@ describe("WorkspaceShell", () => { ).toBeTruthy(); }); + it("shows the first ready document chunks on workspace load", async () => { + const fetch = vi.fn(async (input) => { + const url = getRequestURL(input); + + if (url.pathname === "/api/sources/source_1/chunks") { + return Response.json({ + chunks: [ + { + chunkId: "source_1:chunk_1", + documentId: "doc_1", + sectionPath: "Overview", + type: "text", + content: "First document chunk content.", + sourceTitle: "first.pdf", + }, + ], + pagination: { + page: Number(url.searchParams.get("page") ?? "1"), + pageSize: 100, + total: 1, + totalPages: 1, + }, + }); + } + + return Response.json({ message: "Unexpected request" }, { status: 404 }); + }); + vi.stubGlobal("fetch", fetch); + + render( + React.createElement(C, { + sources: [ + { + id: "source_1", + title: "first.pdf", + status: "ready", + documentId: "doc_1", + }, + { + id: "source_2", + title: "second.pdf", + status: "ready", + documentId: "doc_2", + }, + ], + }), + ); + + const desktopChunksPanel = within(screen.getByTestId("desktop-chunks-panel")); + await waitFor(() => { + expect( + desktopChunksPanel.getByText("First document chunk content."), + ).toBeTruthy(); + }); + expect(countFetches(fetch, "/api/sources/source_1/chunks")).toBe(1); + expect(countFetches(fetch, "/api/sources/source_2/chunks")).toBe(0); + }); + it("focuses guest citations on desktop using loaded demo chunks", async () => { const fetch = vi.fn(async (input) => { const url = getRequestURL(input); diff --git a/src/components/workspace-source-state.test.ts b/src/components/workspace-source-state.test.ts index ccae9e8..7f1596e 100644 --- a/src/components/workspace-source-state.test.ts +++ b/src/components/workspace-source-state.test.ts @@ -5,6 +5,29 @@ import { workspaceSourceState } from "./workspace-source-state"; import type { SourceView } from "@/domains/sources/types"; describe("workspaceSourceState", () => { + it("selects the first ready Source as the initial Source", () => { + const sources: readonly SourceView[] = [ + { + id: "source_parsing", + title: "pending.pdf", + status: "parsing", + mimeType: "application/pdf", + excludedFromQuery: false, + }, + { + id: "source_ready", + title: "ready.pdf", + status: "ready", + mimeType: "application/pdf", + excludedFromQuery: false, + }, + ]; + + expect(workspaceSourceState.getInitialSelectedSourceId(sources)).toBe( + "source_ready", + ); + }); + it("applies source query exclusions without mutating the source list", () => { const sources: readonly SourceView[] = [ { @@ -40,10 +63,27 @@ describe("workspaceSourceState", () => { expect(sources[1]?.excludedFromQuery).toBe(false); }); - it("clears selected and exclusion state when the selected source is archived", () => { + it("moves selection to the first remaining ready Source when the selected Source is archived", () => { + const sources: readonly SourceView[] = [ + { + id: "source_1", + title: "selected.pdf", + status: "ready", + mimeType: "application/pdf", + excludedFromQuery: false, + }, + { + id: "source_2", + title: "remaining.pdf", + status: "ready", + mimeType: "application/pdf", + excludedFromQuery: false, + }, + ]; const result = workspaceSourceState.archiveSource({ sourceId: "source_1", selectedSourceId: "source_1", + sources, sourceExclusionById: { source_1: true, source_2: false, @@ -51,7 +91,7 @@ describe("workspaceSourceState", () => { }); expect(result).toEqual({ - selectedSourceId: null, + selectedSourceId: "source_2", sourceExclusionById: { source_2: false, }, diff --git a/src/components/workspace-source-state.ts b/src/components/workspace-source-state.ts index 9f24185..d4f82d1 100644 --- a/src/components/workspace-source-state.ts +++ b/src/components/workspace-source-state.ts @@ -5,6 +5,7 @@ type SourceExclusionState = Readonly> type ArchiveSourceInput = { readonly sourceId: string readonly selectedSourceId: string | null + readonly sources: readonly SourceView[] readonly sourceExclusionById: SourceExclusionState } @@ -14,9 +15,15 @@ type ArchiveSourceResult = { } type WorkspaceSourceStateModule = { + readonly getFirstReadySourceId: ( + sources: readonly SourceView[], + ) => string | null readonly getInitialSelectedSourceId: ( sources: readonly SourceView[], - isGuest: boolean, + ) => string | null + readonly getResolvedSelectedSourceId: ( + sources: readonly SourceView[], + selectedSourceId: string | null, ) => string | null readonly applyQueryExclusions: ( sources: readonly SourceView[], @@ -38,13 +45,22 @@ type WorkspaceSourceStateModule = { ) => Record } -function getInitialSelectedSourceId( +function getInitialSelectedSourceId(sources: readonly SourceView[]): string | null { + return getFirstReadySourceId(sources) +} + +function getFirstReadySourceId(sources: readonly SourceView[]): string | null { + return sources.find((source) => source.status === "ready")?.id ?? null +} + +function getResolvedSelectedSourceId( sources: readonly SourceView[], - isGuest: boolean, + selectedSourceId: string | null, ): string | null { - if (!isGuest) return null + const selectedSource = sources.find((source) => source.id === selectedSourceId) + if (selectedSource?.status === "ready") return selectedSource.id - return sources.find((source) => source.status === "ready")?.id ?? null + return getFirstReadySourceId(sources) } function applyQueryExclusions( @@ -66,9 +82,14 @@ function upsertSource( } function archiveSource(input: ArchiveSourceInput): ArchiveSourceResult { + const remainingSources = input.sources.filter( + (source) => source.id !== input.sourceId, + ) return { selectedSourceId: - input.selectedSourceId === input.sourceId ? null : input.selectedSourceId, + input.selectedSourceId === input.sourceId + ? getFirstReadySourceId(remainingSources) + : getResolvedSelectedSourceId(remainingSources, input.selectedSourceId), sourceExclusionById: removeRecordKey( input.sourceExclusionById, input.sourceId, @@ -96,7 +117,9 @@ function removeRecordKey( } export const workspaceSourceState: WorkspaceSourceStateModule = { + getFirstReadySourceId, getInitialSelectedSourceId, + getResolvedSelectedSourceId, applyQueryExclusions, upsertSource, archiveSource, diff --git a/src/components/workspace-source-workflow.test.ts b/src/components/workspace-source-workflow.test.ts index 0960704..563f5f0 100644 --- a/src/components/workspace-source-workflow.test.ts +++ b/src/components/workspace-source-workflow.test.ts @@ -53,7 +53,7 @@ describe("useWorkspaceSourceWorkflow", () => { expect(mocks.archiveSource).toHaveBeenCalledWith("source_1") await waitFor(() => { - expect(result.current.selectedSourceId).toBeNull() + expect(result.current.selectedSourceId).toBe("source_2") }) expect(result.current.sources.map((source) => source.id)).toEqual([ "source_2", diff --git a/src/components/workspace-source-workflow.ts b/src/components/workspace-source-workflow.ts index c8f1205..b31380b 100644 --- a/src/components/workspace-source-workflow.ts +++ b/src/components/workspace-source-workflow.ts @@ -41,7 +41,6 @@ export function useWorkspaceSourceWorkflow({ const initialSourceRows = useMemo(() => [...initialSources], [initialSources]) const initialSelectedSourceId = workspaceSourceState.getInitialSelectedSourceId( initialSourceRows, - isGuest, ) const [selectedSourceId, setSelectedSourceId] = useState( initialSelectedSourceId, @@ -68,6 +67,11 @@ export function useWorkspaceSourceWorkflow({ sourceRows, sourceExclusionById, ) + const resolvedSelectedSourceId = + workspaceSourceState.getResolvedSelectedSourceId( + sourceRows, + selectedSourceId, + ) const sourceTitlesByDocumentId = useMemo>>( () => Object.fromEntries( @@ -142,6 +146,7 @@ export function useWorkspaceSourceWorkflow({ workspaceSourceState.archiveSource({ sourceId, selectedSourceId: current, + sources: sourceRows, sourceExclusionById, }).selectedSourceId, ) @@ -149,6 +154,7 @@ export function useWorkspaceSourceWorkflow({ workspaceSourceState.archiveSource({ sourceId, selectedSourceId, + sources: sourceRows, sourceExclusionById: current, }).sourceExclusionById, ) @@ -169,7 +175,7 @@ export function useWorkspaceSourceWorkflow({ handleSourceUploaded, handleToggleIncluded, readySourceCount, - selectedSourceId, + selectedSourceId: resolvedSelectedSourceId, setSelectedSourceId, sourceTitlesByDocumentId, sources, From a3dbadd52ac3d0eaa4aab8058d58b0eb79bad6d0 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 02:51:20 +0800 Subject: [PATCH 04/16] fix: make text original previews responsive --- .../source-original-preview.test.ts | 34 +++++++++++++++++++ src/components/source-original-preview.tsx | 21 ++++++++++-- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/src/components/source-original-preview.test.ts b/src/components/source-original-preview.test.ts index c673793..0ceb1e9 100644 --- a/src/components/source-original-preview.test.ts +++ b/src/components/source-original-preview.test.ts @@ -566,6 +566,40 @@ describe("SourceOriginalPreview", () => { expect(screen.queryByText(/
/)).toBeNull(); }); + it("keeps Markdown content readable inside the responsive original preview shell", async () => { + vi.stubGlobal( + "fetch", + vi.fn(() => + Promise.resolve( + new Response("# Scan report\n\nThe scan found placeholder keys.", { + status: 200, + }), + ), + ), + ); + + render( + React.createElement(SourceOriginalPreview, { + sourceTitle: "scan-report.md", + file: { + url: "https://store.public.blob.vercel-storage.com/source-uploads/upload_1/scan-report.md", + mimeType: "text/markdown", + }, + }), + ); + + await waitFor(() => { + expect(screen.getByText("Scan report")).toBeTruthy(); + }); + + const previewShell = screen.getByTestId("source-original-preview"); + expect(previewShell.className).toContain("w-[90%]"); + expect(previewShell.className).toContain("max-w-[1600px]"); + + const markdownPreview = document.querySelector(".original-markdown-preview"); + expect(markdownPreview?.parentElement?.className).toContain("max-w-4xl"); + }); + it("hides the download action for non-downloadable demo originals", () => { render( React.createElement(SourceOriginalPreview, { diff --git a/src/components/source-original-preview.tsx b/src/components/source-original-preview.tsx index a105abe..03c69a1 100644 --- a/src/components/source-original-preview.tsx +++ b/src/components/source-original-preview.tsx @@ -49,7 +49,7 @@ export function SourceOriginalPreview({
@@ -84,6 +84,10 @@ export function SourceOriginalPreview({ ); } +function getPreviewShellClassName(): string { + return "mx-auto flex w-[90%] min-w-0 max-w-[1600px] flex-col gap-3 p-3 sm:p-6"; +} + function renderPreview( kind: PreviewKind, sourceTitle: string, @@ -119,9 +123,9 @@ function renderPreview( /> ); case "markdown": - return ; + return renderReadingPreview(file, "markdown"); case "text": - return ; + return renderReadingPreview(file, "text"); case "docx": return ; case "unsupported": @@ -129,6 +133,17 @@ function renderPreview( } } +function renderReadingPreview( + file: SourceOriginalFileView, + variant: "markdown" | "text", +): ReactNode { + return ( +
+ +
+ ); +} + function UnsupportedPreview(): ReactNode { return (
From cc3c7dc657e63347d5d813c0e063b8d97c7d9475 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 09:39:11 +0800 Subject: [PATCH 05/16] fix: load materialized demo chunks from demo api --- .../sources/[sourceId]/chunks/route.test.ts | 90 +++++++++++++++++++ src/domains/sources/route-chunks.ts | 15 +++- 2 files changed, 101 insertions(+), 4 deletions(-) diff --git a/src/app/api/sources/[sourceId]/chunks/route.test.ts b/src/app/api/sources/[sourceId]/chunks/route.test.ts index 7eec12f..3a7adb3 100644 --- a/src/app/api/sources/[sourceId]/chunks/route.test.ts +++ b/src/app/api/sources/[sourceId]/chunks/route.test.ts @@ -275,6 +275,96 @@ describe("GET /api/sources/[sourceId]/chunks", () => { expect(mocks.getSourceParseAssetUrls).not.toHaveBeenCalled() }) + it("serves demo chunks for authenticated materialized demo sources", async () => { + mocks.getCurrentUser.mockResolvedValue({ + id: "user_1", + email: null, + name: null, + }) + mocks.ensureWorkspace.mockResolvedValue({ + id: "workspace_1", + userId: "user_1", + namespace: "notebook-workspace_1", + createdAt: new Date("2026-05-10T00:00:00.000Z"), + }) + mocks.findSourceInWorkspace.mockResolvedValue({ + id: "source_materialized_demo", + workspaceId: "workspace_1", + title: "TSLA-Q4-2025-Update.pdf", + mimeType: "application/pdf", + sizeBytes: 1024, + status: "ready", + failureReason: null, + knowhereJobId: null, + knowhereDocumentId: "copied-doc-tsla-q4-2025", + stagedBlobPathname: null, + stagedBlobUrl: null, + originalBlobPathname: null, + originalBlobUrl: "/api/demo-sources/demo-tsla-q4-2025/original", + demoKey: "demo-tsla-q4-2025", + createdAt: new Date("2026-05-10T00:00:00.000Z"), + updatedAt: new Date("2026-05-10T00:00:00.000Z"), + deletedAt: null, + }) + mocks.fetchDemoChunkPage.mockResolvedValue({ + demoSourceId: "demo-tsla-q4-2025", + canonicalDocumentId: "demo-doc-tsla-q4-2025", + title: "TSLA-Q4-2025-Update.pdf", + mimeType: "application/pdf", + chunks: [ + { + id: "demo-tsla-q4-2025:chunk_1", + chunkId: "chunk_1", + chunkType: "text", + content: "Tesla demo content", + sectionPath: "Summary", + sourceChunkPath: "Summary", + filePath: null, + sortOrder: 0, + metadata: {}, + assetUrl: null, + }, + ], + pagination: { + page: 1, + pageSize: 100, + total: 70, + totalPages: 1, + }, + }) + + const response = await GET( + new NextRequest( + "http://localhost:3001/api/sources/source_materialized_demo/chunks?page=1&pageSize=100", + ), + { params: Promise.resolve({ sourceId: "source_materialized_demo" }) }, + ) + + await expect(response.json()).resolves.toMatchObject({ + chunks: [ + { + chunkId: "demo-tsla-q4-2025:chunk_1", + documentId: "demo-doc-tsla-q4-2025", + sourceTitle: "TSLA-Q4-2025-Update.pdf", + }, + ], + pagination: { + page: 1, + pageSize: 100, + total: 70, + }, + }) + expect(response.status).toBe(200) + expect(mocks.fetchDemoChunkPage).toHaveBeenCalledWith({ + demoSourceId: "demo-tsla-q4-2025", + page: 1, + pageSize: 100, + }) + expect(mocks.ensureApiKeyForWorkspace).not.toHaveBeenCalled() + expect(mocks.makeKnowhereClient).not.toHaveBeenCalled() + expect(mocks.getSourceParseAssetUrls).not.toHaveBeenCalled() + }) + it("logs the demo chunk load failure before returning 404", async () => { const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined) try { diff --git a/src/domains/sources/route-chunks.ts b/src/domains/sources/route-chunks.ts index 8666422..2e7bad9 100644 --- a/src/domains/sources/route-chunks.ts +++ b/src/domains/sources/route-chunks.ts @@ -56,6 +56,10 @@ async function loadSourceChunks( return (await loadDemoChunkPage(input, deps)) ?? sourceNotFound() } + if (source.demoKey) { + return (await loadDemoChunkPage(input, deps, source.demoKey)) ?? sourceNotFound() + } + const client = await getClientForWorkspace( workspace.id, input.cookieHeader, @@ -84,13 +88,14 @@ async function loadSourceChunks( async function loadDemoChunkPage( input: LoadSourceChunksInput, deps: RouteChunksDependencies, + demoSourceId: string = input.sourceId, ): Promise | null> { try { const pages = input.shouldLoadAll - ? await loadAllDemoChunkPages(input, deps) + ? await loadAllDemoChunkPages(input, deps, demoSourceId) : [ await deps.demoApi.fetchChunkPage({ - demoSourceId: input.sourceId, + demoSourceId, page: input.pageParams.page, pageSize: input.pageParams.pageSize, }), @@ -123,6 +128,7 @@ async function loadDemoChunkPage( } catch (error) { logger.warn("sources: demo chunk load failed", { sourceId: input.sourceId, + demoSourceId, page: input.pageParams.page, pageSize: input.pageParams.pageSize, shouldLoadAll: input.shouldLoadAll, @@ -136,10 +142,11 @@ async function loadDemoChunkPage( async function loadAllDemoChunkPages( input: LoadSourceChunksInput, deps: RouteChunksDependencies, + demoSourceId: string, ): Promise { const pageSize = 200 const firstPage = await deps.demoApi.fetchChunkPage({ - demoSourceId: input.sourceId, + demoSourceId, page: 1, pageSize, }) @@ -151,7 +158,7 @@ async function loadAllDemoChunkPages( ) { pages.push( await deps.demoApi.fetchChunkPage({ - demoSourceId: input.sourceId, + demoSourceId, page: pageNumber, pageSize, }), From a27a62f93aa6076327d7fc67797fd985f16383bd Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 09:47:32 +0800 Subject: [PATCH 06/16] fix: avoid copied demo chunk counts --- .../demo/workspace-source-resolution.ts | 30 ++++++++ src/domains/sources/counts.test.ts | 9 +-- src/domains/sources/counts.ts | 5 +- src/domains/sources/route-listing.ts | 23 ++++++- src/domains/sources/route-service.test.ts | 69 +++++++++++++++++++ src/domains/workspace/initial-state.test.ts | 4 ++ src/domains/workspace/initial-state.ts | 23 ++++++- 7 files changed, 150 insertions(+), 13 deletions(-) diff --git a/src/domains/demo/workspace-source-resolution.ts b/src/domains/demo/workspace-source-resolution.ts index e14339f..39abc5a 100644 --- a/src/domains/demo/workspace-source-resolution.ts +++ b/src/domains/demo/workspace-source-resolution.ts @@ -6,6 +6,10 @@ type WorkspaceDemoSourceResolution = { readonly workspaceSources: readonly Source[] } +type SourceViewOptions = { + readonly chunkCount?: number +} + export function resolveWorkspaceDemoSources( sources: readonly Source[], catalog: DemoCatalog, @@ -35,6 +39,32 @@ export function resolveWorkspaceDemoSources( } } +export function getWorkspaceSourcesNeedingKnowhereChunkCount( + sources: readonly Source[], +): Source[] { + return sources.filter((source) => !source.demoKey) +} + +export function getMaterializedDemoSourceViewOptionsBySourceId( + sources: readonly Source[], + catalog: DemoCatalog, +): ReadonlyMap { + const chunkCountByDemoSourceId: ReadonlyMap = new Map( + catalog.sources.map((source) => [source.demoSourceId, source.chunkCount]), + ) + + return new Map( + sources.flatMap((source): readonly [string, SourceViewOptions][] => { + if (!source.demoKey) return [] + + const chunkCount = chunkCountByDemoSourceId.get(source.demoKey) + if (chunkCount === undefined) return [] + + return [[source.id, { chunkCount }]] + }), + ) +} + function isLegacyCanonicalDemoSource( source: Source, canonicalDocumentIdByDemoSourceId: ReadonlyMap, diff --git a/src/domains/sources/counts.test.ts b/src/domains/sources/counts.test.ts index 816c72f..22cdda9 100644 --- a/src/domains/sources/counts.test.ts +++ b/src/domains/sources/counts.test.ts @@ -76,7 +76,7 @@ describe("countChunksBySourceId", () => { expect(counts.size).toBe(0) }) - it("counts materialized demo sources through their copied document id", async () => { + it("does not count materialized demo sources through their copied document id", async () => { const listChunks = vi.fn().mockResolvedValue({ pagination: { total: 70 }, }) @@ -99,10 +99,7 @@ describe("countChunksBySourceId", () => { ), ) - expect(listChunks).toHaveBeenCalledWith("doc_user_copy", { - page: 1, - pageSize: 1, - }) - expect(counts).toEqual(new Map([["source_demo", 70]])) + expect(listChunks).not.toHaveBeenCalled() + expect(counts.size).toBe(0) }) }) diff --git a/src/domains/sources/counts.ts b/src/domains/sources/counts.ts index 9419714..310a694 100644 --- a/src/domains/sources/counts.ts +++ b/src/domains/sources/counts.ts @@ -11,7 +11,10 @@ export const countChunksBySourceId = ( ) => Effect.gen(function* () { const readySources = sources.filter( - (source) => source.status === "ready" && source.knowhereDocumentId, + (source) => + !source.demoKey && + source.status === "ready" && + source.knowhereDocumentId, ) if (readySources.length === 0) return new Map() diff --git a/src/domains/sources/route-listing.ts b/src/domains/sources/route-listing.ts index 95adbb3..1255136 100644 --- a/src/domains/sources/route-listing.ts +++ b/src/domains/sources/route-listing.ts @@ -1,7 +1,11 @@ import { Effect } from "effect" import { demoView } from "@/domains/demo/view" -import { resolveWorkspaceDemoSources } from "@/domains/demo/workspace-source-resolution" +import { + getMaterializedDemoSourceViewOptionsBySourceId, + getWorkspaceSourcesNeedingKnowhereChunkCount, + resolveWorkspaceDemoSources, +} from "@/domains/demo/workspace-source-resolution" import { routeResult } from "@/lib/route-result" import type { DemoCatalog } from "@/integrations/knowhere-demo" import { toSourceView } from "./view" @@ -62,9 +66,18 @@ async function listSources( ) const sources = await deps.reconcileSourcesForWorkspace(workspace, client) const demoSourceResolution = resolveWorkspaceDemoSources(sources, catalog) + const sourcesNeedingKnowhereChunkCount = + getWorkspaceSourcesNeedingKnowhereChunkCount( + demoSourceResolution.workspaceSources, + ) + const materializedDemoSourceOptions = + getMaterializedDemoSourceViewOptionsBySourceId( + demoSourceResolution.workspaceSources, + catalog, + ) const sourceOptions = await Effect.runPromise( deps.getSourceViewOptionsBySourceId( - demoSourceResolution.workspaceSources, + sourcesNeedingKnowhereChunkCount, client, ), ) @@ -83,7 +96,11 @@ async function listSources( sources: [ ...visibleDemoSources, ...demoSourceResolution.workspaceSources.map((source) => - toSourceView(source, sourceOptions.get(source.id)), + toSourceView( + source, + materializedDemoSourceOptions.get(source.id) ?? + sourceOptions.get(source.id), + ), ), ], }) diff --git a/src/domains/sources/route-service.test.ts b/src/domains/sources/route-service.test.ts index abaa06c..7276d29 100644 --- a/src/domains/sources/route-service.test.ts +++ b/src/domains/sources/route-service.test.ts @@ -320,6 +320,75 @@ describe("source route service", () => { }); }); + it("uses demo catalog counts for materialized demo sources", async () => { + const materializedSource: Source = { + ...source, + id: "source_demo", + title: "TSLA-Q4-2025-Update.pdf", + status: "ready", + demoKey: "demo-tsla-q4-2025", + knowhereJobId: null, + knowhereDocumentId: "doc_user_copy", + originalBlobUrl: "/api/demo-sources/demo-tsla-q4-2025/original", + }; + const knowhereClient = { + documents: { + archive: vi.fn(async () => undefined), + listChunks: vi.fn(async () => ({ + chunks: [], + pagination: { + page: 1, + pageSize: 1, + total: 0, + totalPages: 0, + }, + })), + }, + jobs: { + create: vi.fn(), + upload: vi.fn(), + }, + }; + const getSourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map())); + const listing = createRouteListing({ + demoApi: { + fetchCatalog: vi.fn(async () => demoCatalog), + }, + ensureApiKeyForWorkspace: vi.fn(async () => "jwt_123"), + ensureWorkspace: vi.fn(async () => workspace), + getCurrentUser: vi.fn(async () => ({ + id: "user_1", + email: null, + name: null, + })), + getSourceViewOptionsBySourceId, + makeKnowhereClient: vi.fn(() => knowhereClient), + reconcileSourcesForWorkspace: vi.fn(async () => [materializedSource]), + sourceService: { listHiddenDemoSourceIds: vi.fn(async () => []) }, + }); + + const result = await listing.listSources({ cookieHeader: "session=abc" }); + + expect(getSourceViewOptionsBySourceId).toHaveBeenCalledWith( + [], + knowhereClient, + ); + expect(knowhereClient.documents.listChunks).not.toHaveBeenCalled(); + expect(result).toEqual({ + status: 200, + body: { + sources: [ + expect.objectContaining({ + id: "source_demo", + kind: "workspace", + documentId: "doc_user_copy", + chunkCount: 70, + }), + ], + }, + }); + }); + it("lists API-owned demo sources for anonymous users", async () => { const ensureWorkspace = vi.fn(async () => workspace); const service = createSourceRouteService({ diff --git a/src/domains/workspace/initial-state.test.ts b/src/domains/workspace/initial-state.test.ts index 88cdf5d..47f48a7 100644 --- a/src/domains/workspace/initial-state.test.ts +++ b/src/domains/workspace/initial-state.test.ts @@ -174,18 +174,22 @@ describe("loadWorkspaceShellInitialState", () => { title: "TSLA-Q4-2025-Update.pdf", knowhereDocumentId: "doc_user_copy", }) + const sourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map())) const deps = createDependencies({ listHiddenDemoSourceIds: vi.fn(async () => ["another-demo"]), reconcileSourcesForWorkspace: vi.fn(async () => [materializedSource]), + sourceViewOptionsBySourceId, }) const state = await loadWorkspaceShellInitialState(deps) + expect(sourceViewOptionsBySourceId).toHaveBeenCalledWith([], expect.any(Object)) expect(state.sources).toEqual([ expect.objectContaining({ id: "source_demo", kind: "workspace", documentId: "doc_user_copy", + chunkCount: 70, }), ]) }) diff --git a/src/domains/workspace/initial-state.ts b/src/domains/workspace/initial-state.ts index e1a5e82..574006a 100644 --- a/src/domains/workspace/initial-state.ts +++ b/src/domains/workspace/initial-state.ts @@ -4,7 +4,11 @@ import { Effect } from "effect" import type { ChatMessageView } from "@/domains/chat/types" import { demoView } from "@/domains/demo/view" -import { resolveWorkspaceDemoSources } from "@/domains/demo/workspace-source-resolution" +import { + getMaterializedDemoSourceViewOptionsBySourceId, + getWorkspaceSourcesNeedingKnowhereChunkCount, + resolveWorkspaceDemoSources, +} from "@/domains/demo/workspace-source-resolution" import { chatThreadService } from "@/domains/chat/thread-service" import { toChatMessageView, toChatThreadView } from "@/domains/chat/view" import { sourceViewOptionsBySourceId as getSourceViewOptionsBySourceId } from "@/domains/sources/counts" @@ -147,8 +151,17 @@ export async function loadWorkspaceShellInitialState( (message) => toChatMessageView(message), ) : [] + const sourcesNeedingKnowhereChunkCount = + getWorkspaceSourcesNeedingKnowhereChunkCount( + demoSourceResolution.workspaceSources, + ) + const materializedDemoSourceOptions = + getMaterializedDemoSourceViewOptionsBySourceId( + demoSourceResolution.workspaceSources, + demoCatalog, + ) const sourceOptions = await Effect.runPromise( - deps.sourceViewOptionsBySourceId(demoSourceResolution.workspaceSources, client), + deps.sourceViewOptionsBySourceId(sourcesNeedingKnowhereChunkCount, client), ) return { @@ -165,7 +178,11 @@ export async function loadWorkspaceShellInitialState( sources: [ ...demoSources, ...demoSourceResolution.workspaceSources.map((source) => - toSourceView(source, sourceOptions.get(source.id)), + toSourceView( + source, + materializedDemoSourceOptions.get(source.id) ?? + sourceOptions.get(source.id), + ), ), ], chatThreads: chatThreads.map(toChatThreadView), From 9ee238eb52520082904cfe2926b386db4090104e Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 14:42:50 +0800 Subject: [PATCH 07/16] fix: remove chunk search, reduce page size to 50, fix logo warning --- src/components/chunks-panel-state.test.ts | 32 --- src/components/chunks-panel-state.ts | 64 ------ src/components/chunks-panel.tsx | 227 ---------------------- src/components/notebook-logo-mark.tsx | 1 + src/components/parsed-chunk-card.test.ts | 4 - src/components/parsed-chunk-card.tsx | 80 ++------ src/domains/chunks/index.ts | 2 +- src/domains/workspace/client.ts | 2 +- 8 files changed, 16 insertions(+), 396 deletions(-) diff --git a/src/components/chunks-panel-state.test.ts b/src/components/chunks-panel-state.test.ts index 81c17a8..114ee4c 100644 --- a/src/components/chunks-panel-state.test.ts +++ b/src/components/chunks-panel-state.test.ts @@ -93,36 +93,4 @@ describe("chunksPanelState", () => { ).toBe("Image 12") }) - it("finds chunk search matches across content, summary, and keywords", () => { - const chunks: ParsedChunkView[] = [ - { - chunkId: "chunk_1", - type: "text", - content: "Tesla storage revenue increased while Tesla deployed Megapack.", - summary: "Storage update", - keywords: ["Energy storage"], - sourceTitle: "report.pdf", - }, - { - chunkId: "chunk_2", - type: "image", - content: "", - summary: "Megapack deployment chart", - keywords: ["Tesla Energy"], - sourceTitle: "report.pdf", - }, - { - chunkId: "chunk_3", - type: "text", - content: "Vehicle deliveries only.", - sourceTitle: "report.pdf", - }, - ] - - expect(chunksPanelState.getChunkSearchMatches(chunks, " tesla ")).toEqual([ - { chunkId: "chunk_1", matchCount: 2 }, - { chunkId: "chunk_2", matchCount: 1 }, - ]) - expect(chunksPanelState.getChunkSearchMatches(chunks, "missing")).toEqual([]) - }) }) diff --git a/src/components/chunks-panel-state.ts b/src/components/chunks-panel-state.ts index d1b101b..430951a 100644 --- a/src/components/chunks-panel-state.ts +++ b/src/components/chunks-panel-state.ts @@ -9,19 +9,10 @@ type RenderableReference = { readonly connection: ParsedChunkConnection } -export type ChunkSearchMatch = { - readonly chunkId: string - readonly matchCount: number -} - type ChunksPanelStateModule = { readonly formatChunkSectionPath: ( sectionPath: ParsedChunkView["sectionPath"], ) => string | null - readonly getChunkSearchMatches: ( - chunks: readonly ParsedChunkView[], - query: string, - ) => readonly ChunkSearchMatch[] readonly formatReferenceLabel: (ref: string) => string readonly getChunksWithFocusedFirst: ( chunks: readonly ParsedChunkView[], @@ -31,7 +22,6 @@ type ChunksPanelStateModule = { readonly getRenderableReferences: ( chunk: ParsedChunkView, ) => RenderableReference[] - readonly normalizeChunkSearchQuery: (query: string) => string } function getChunksWithFocusedFirst( @@ -139,58 +129,6 @@ function getRenderableReferences( return nonOverlapping } -function getChunkSearchMatches( - chunks: readonly ParsedChunkView[], - query: string, -): readonly ChunkSearchMatch[] { - const normalizedQuery = normalizeChunkSearchQuery(query) - if (!normalizedQuery) return [] - - return chunks.flatMap((chunk): ChunkSearchMatch[] => { - const matchCount = countChunkSearchMatches(chunk, normalizedQuery) - if (matchCount === 0) return [] - return [{ chunkId: chunk.chunkId, matchCount }] - }) -} - -function normalizeChunkSearchQuery(query: string): string { - return query.trim().replace(/\s+/g, " ").toLocaleLowerCase() -} - -function countChunkSearchMatches( - chunk: ParsedChunkView, - normalizedQuery: string, -): number { - return getChunkSearchText(chunk).reduce( - (total, text) => total + countTextMatches(text, normalizedQuery), - 0, - ) -} - -function getChunkSearchText(chunk: ParsedChunkView): readonly string[] { - return [ - chunk.content, - chunk.summary ?? "", - ...(chunk.keywords ?? []), - ].filter((text) => text.trim().length > 0) -} - -function countTextMatches(text: string, normalizedQuery: string): number { - const normalizedText = text.toLocaleLowerCase() - let count = 0 - let cursor = 0 - - while (cursor < normalizedText.length) { - const matchIndex = normalizedText.indexOf(normalizedQuery, cursor) - if (matchIndex < 0) return count - - count += 1 - cursor = matchIndex + normalizedQuery.length - } - - return count -} - function getReferenceRange( content: string, connection: ParsedChunkConnection, @@ -253,10 +191,8 @@ function capitalize(value: string): string { export const chunksPanelState: ChunksPanelStateModule = { formatChunkSectionPath, - getChunkSearchMatches, formatReferenceLabel, getChunksWithFocusedFirst, getReferenceLabel, getRenderableReferences, - normalizeChunkSearchQuery, } diff --git a/src/components/chunks-panel.tsx b/src/components/chunks-panel.tsx index 85c6714..6ea3684 100644 --- a/src/components/chunks-panel.tsx +++ b/src/components/chunks-panel.tsx @@ -2,32 +2,21 @@ import { type CSSProperties, - type ChangeEvent, type ReactNode, useCallback, useEffect, - useId, - useMemo, useState, } from "react"; import { type VirtualItem } from "@tanstack/react-virtual"; import { - ChevronDown, - ChevronUp, FilePlus2, Layers, - Search, UploadCloud, - X, } from "lucide-react"; import { ScrollArea } from "@/components/ui/scroll-area"; import { SourceOriginalPreview } from "@/components/source-original-preview"; import { SourceUploadDialog } from "@/components/source-upload-dialog"; import { useChunksPanelWorkflow } from "@/components/chunks-panel-workflow"; -import { - chunksPanelState, - type ChunkSearchMatch, -} from "@/components/chunks-panel-state"; import { ParsedChunkCard } from "@/components/parsed-chunk-card"; import { useSourceOriginalPreviewWarmup } from "@/components/source-original-preview-warmup"; import { sourceOriginalPreviewModel } from "@/components/source-original-preview-model"; @@ -71,32 +60,6 @@ export function ChunksPanel({ const [mountedOriginalPreviewKey, setMountedOriginalPreviewKey] = useState< string | null >(null); - const [searchQuery, setSearchQuery] = useState(""); - const [activeSearchMatchIndex, setActiveSearchMatchIndex] = useState(-1); - const normalizedSearchQuery = chunksPanelState.normalizeChunkSearchQuery( - searchQuery, - ); - const searchMatches = useMemo( - () => chunksPanelState.getChunkSearchMatches(chunks, searchQuery), - [chunks, searchQuery], - ); - const totalSearchHitCount = useMemo( - () => - searchMatches.reduce( - (total, searchMatch) => total + searchMatch.matchCount, - 0, - ), - [searchMatches], - ); - const effectiveActiveSearchMatchIndex = getEffectiveSearchMatchIndex( - activeSearchMatchIndex, - searchMatches, - normalizedSearchQuery, - ); - const activeSearchMatch = - effectiveActiveSearchMatchIndex >= 0 - ? (searchMatches[effectiveActiveSearchMatchIndex] ?? null) - : null; const { activeFocusedChunkId, handleChunkSelected: selectChunk, @@ -125,20 +88,6 @@ export function ChunksPanel({ onLoadMore, }); - useEffect(() => { - if (!normalizedSearchQuery) { - requestChunkFocus(null); - return; - } - - if (!activeSearchMatch) { - requestChunkFocus(null); - return; - } - - requestChunkFocus(activeSearchMatch.chunkId); - }, [activeSearchMatch, normalizedSearchQuery, requestChunkFocus]); - useSourceOriginalPreviewWarmup({ sourceTitle: selectedSource, file: selectedSourceFile, @@ -161,31 +110,6 @@ export function ChunksPanel({ rememberOriginalPreview(); selectOriginalView(); }, [rememberOriginalPreview, selectOriginalView]); - const handleSearchQueryChange = useCallback( - (query: string): void => { - const nextMatches = chunksPanelState.getChunkSearchMatches(chunks, query); - setSearchQuery(query); - setActiveSearchMatchIndex(nextMatches.length > 0 ? 0 : -1); - if (visibleView === "original") handleParsedViewSelected(); - }, - [chunks, handleParsedViewSelected, visibleView], - ); - const handleSearchCleared = useCallback((): void => { - setSearchQuery(""); - setActiveSearchMatchIndex(-1); - requestChunkFocus(null); - }, [requestChunkFocus]); - const handlePreviousSearchMatch = useCallback((): void => { - setActiveSearchMatchIndex((currentIndex) => - getRelativeSearchMatchIndex(currentIndex, searchMatches, -1), - ); - }, [searchMatches]); - const handleNextSearchMatch = useCallback((): void => { - setActiveSearchMatchIndex((currentIndex) => - getRelativeSearchMatchIndex(currentIndex, searchMatches, 1), - ); - }, [searchMatches]); - const headerTitle = focusedChunkId ? "Referenced Chunks" : "Parsed Chunks"; const shouldMountOriginalPreview = visibleView === "original" || @@ -230,20 +154,6 @@ export function ChunksPanel({

- = 0 - ? effectiveActiveSearchMatchIndex + 1 - : 0 - } - matchCount={searchMatches.length} - query={searchQuery} - totalHitCount={totalSearchHitCount} - onClear={handleSearchCleared} - onNext={handleNextSearchMatch} - onPrevious={handlePreviousSearchMatch} - onQueryChange={handleSearchQueryChange} - /> {hasOriginalFile ? (
- ) : null} -
- - {resultLabel} - -
- - -
-
- ); -} - -function getSearchResultLabel({ - activeMatchOrdinal, - hasQuery, - matchCount, - totalHitCount, -}: { - readonly activeMatchOrdinal: number; - readonly hasQuery: boolean; - readonly matchCount: number; - readonly totalHitCount: number; -}): string { - if (!hasQuery) return "Search chunks"; - if (matchCount === 0) return "No matches"; - - const hitLabel = totalHitCount === 1 ? "hit" : "hits"; - return `${activeMatchOrdinal}/${matchCount} chunks · ${totalHitCount} ${hitLabel}`; -} - -function getRelativeSearchMatchIndex( - currentIndex: number, - matches: readonly ChunkSearchMatch[], - delta: number, -): number { - if (matches.length === 0) return -1; - - const startingIndex = currentIndex >= 0 ? currentIndex : 0; - return (startingIndex + delta + matches.length) % matches.length; -} - -function getEffectiveSearchMatchIndex( - currentIndex: number, - matches: readonly ChunkSearchMatch[], - normalizedSearchQuery: string, -): number { - if (!normalizedSearchQuery || matches.length === 0) return -1; - if (currentIndex < 0) return 0; - return Math.min(currentIndex, matches.length - 1); -} - -const searchNavigationButtonClassName = - "flex size-8 items-center justify-center rounded-md text-muted-foreground transition-colors hover:bg-background hover:text-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-[#8e51ff]/25 disabled:pointer-events-none disabled:opacity-40"; - function EmptySourceUploadState({ onLoginClick, onSourceUploaded, @@ -614,7 +390,6 @@ function VirtualChunkRow({ chunk, focusedChunkId, isOriginalPreviewAvailable, - searchQuery, measureElement, onChunkClick, onReferenceClick, @@ -623,7 +398,6 @@ function VirtualChunkRow({ chunk: ParsedChunkView | undefined; focusedChunkId: string | null; isOriginalPreviewAvailable: boolean; - searchQuery: string; measureElement: (node: HTMLDivElement | null) => void; onChunkClick?: (chunk: ParsedChunkView) => void; onReferenceClick: (chunkId: string) => void; @@ -651,7 +425,6 @@ function VirtualChunkRow({ chunk={chunk} isFocused={chunk.chunkId === focusedChunkId} isOriginalPreviewAvailable={isOriginalPreviewAvailable} - searchQuery={searchQuery} onChunkClick={onChunkClick} onReferenceClick={onReferenceClick} /> diff --git a/src/components/notebook-logo-mark.tsx b/src/components/notebook-logo-mark.tsx index 1b3f58a..79208dc 100644 --- a/src/components/notebook-logo-mark.tsx +++ b/src/components/notebook-logo-mark.tsx @@ -21,6 +21,7 @@ export function NotebookLogoMark({ width, className }: NotebookLogoMarkProps) { className={className} width={width} height={height} + style={{ height: "auto" }} /> ); } diff --git a/src/components/parsed-chunk-card.test.ts b/src/components/parsed-chunk-card.test.ts index 85283f7..350a1fb 100644 --- a/src/components/parsed-chunk-card.test.ts +++ b/src/components/parsed-chunk-card.test.ts @@ -24,7 +24,6 @@ describe("ParsedChunkCard", () => { keywords: ["Supercharging", "AI training capacity"], }, isFocused: true, - searchQuery: "capacity", onReferenceClick: vi.fn(), }), ); @@ -44,9 +43,6 @@ describe("ParsedChunkCard", () => { expect(screen.getByTestId("chunk-card-shell-text_1").className).toContain( "min-w-0", ); - expect( - container.querySelectorAll('mark[data-chunk-search-match="true"]').length, - ).toBeGreaterThan(0); }); it("routes resolved artifact reference clicks to the target chunk", async () => { diff --git a/src/components/parsed-chunk-card.tsx b/src/components/parsed-chunk-card.tsx index 18921fd..a75d41b 100644 --- a/src/components/parsed-chunk-card.tsx +++ b/src/components/parsed-chunk-card.tsx @@ -23,14 +23,12 @@ export function ParsedChunkCard({ chunk, isFocused, isOriginalPreviewAvailable = false, - searchQuery = "", onChunkClick, onReferenceClick, }: { readonly chunk: ParsedChunkView; readonly isFocused: boolean; readonly isOriginalPreviewAvailable?: boolean; - readonly searchQuery?: string; readonly onChunkClick?: (chunk: ParsedChunkView) => void; readonly onReferenceClick: (chunkId: string) => void; }): ReactNode { @@ -41,7 +39,6 @@ export function ParsedChunkCard({ chunk={chunk} isFocused={isFocused} isOriginalPreviewAvailable={isOriginalPreviewAvailable} - searchQuery={searchQuery} onChunkClick={onChunkClick} /> @@ -54,7 +51,6 @@ export function ParsedChunkCard({ chunk={chunk} isFocused={isFocused} isOriginalPreviewAvailable={isOriginalPreviewAvailable} - searchQuery={searchQuery} onChunkClick={onChunkClick} /> @@ -66,7 +62,6 @@ export function ParsedChunkCard({ chunk={chunk} isFocused={isFocused} isOriginalPreviewAvailable={isOriginalPreviewAvailable} - searchQuery={searchQuery} onChunkClick={onChunkClick} onReferenceClick={onReferenceClick} /> @@ -230,10 +225,8 @@ function getOpenOriginalButtonLabel( function ChunkSummaryPanel({ chunk, - searchQuery, }: { readonly chunk: ParsedChunkView; - readonly searchQuery: string; }): ReactNode { if (!chunk.summary) return null; @@ -244,7 +237,7 @@ function ChunkSummaryPanel({ > } label="Summary" />

- {renderHighlightedSearchText(chunk.summary, searchQuery)} + {chunk.summary}

); @@ -270,10 +263,8 @@ function ChunkContentPanel({ function ChunkKeywords({ chunk, - searchQuery, }: { readonly chunk: ParsedChunkView; - readonly searchQuery: string; }): ReactNode { if (!chunk.keywords || chunk.keywords.length === 0) return null; @@ -295,7 +286,7 @@ function ChunkKeywords({ variant="secondary" className={keywordBadgeClassName} > - {renderHighlightedSearchText(keyword, searchQuery)} + {keyword} ))}
@@ -331,14 +322,12 @@ function TextChunkCard({ chunk, isFocused, isOriginalPreviewAvailable, - searchQuery, onChunkClick, onReferenceClick, }: { readonly chunk: ParsedChunkView; readonly isFocused: boolean; readonly isOriginalPreviewAvailable: boolean; - readonly searchQuery: string; readonly onChunkClick?: (chunk: ParsedChunkView) => void; readonly onReferenceClick: (chunkId: string) => void; }): ReactNode { @@ -349,13 +338,13 @@ function TextChunkCard({ isOriginalPreviewAvailable={isOriginalPreviewAvailable} onChunkClick={onChunkClick} > - + 
-          {renderTextChunkContent(chunk, searchQuery, onReferenceClick)}
+          {renderTextChunkContent(chunk, onReferenceClick)}
- + ); } @@ -364,13 +353,11 @@ function ImageChunkCard({ chunk, isFocused, isOriginalPreviewAvailable, - searchQuery, onChunkClick, }: { readonly chunk: ParsedChunkView; readonly isFocused: boolean; readonly isOriginalPreviewAvailable: boolean; - readonly searchQuery: string; readonly onChunkClick?: (chunk: ParsedChunkView) => void; }): ReactNode { return ( @@ -380,7 +367,7 @@ function ImageChunkCard({ isOriginalPreviewAvailable={isOriginalPreviewAvailable} onChunkClick={onChunkClick} > - + {chunk.assetUrl ? (
@@ -400,31 +387,30 @@ function ImageChunkCard({

{chunk.summary - ? renderHighlightedSearchText(chunk.summary, searchQuery) + ? chunk.summary : "Image content is not available in this view."}

)} - + ); } function renderTextChunkContent( chunk: ParsedChunkView, - searchQuery: string, onReferenceClick: (chunkId: string) => void, ): ReactNode { const parts = parsedChunkCardModel.getTextContentParts(chunk); if (parts.length === 1 && parts[0]?.type === "text") { - return renderHighlightedSearchText(parts[0].text, searchQuery); + return parts[0].text; } return parts.map((part) => { if (part.type === "text") { - return renderHighlightedSearchText(part.text, searchQuery); + return part.text; } return ( @@ -464,13 +450,11 @@ function TableChunkCard({ chunk, isFocused, isOriginalPreviewAvailable, - searchQuery, onChunkClick, }: { readonly chunk: ParsedChunkView; readonly isFocused: boolean; readonly isOriginalPreviewAvailable: boolean; - readonly searchQuery: string; readonly onChunkClick?: (chunk: ParsedChunkView) => void; }): ReactNode { const safeHtml = useMemo( @@ -485,7 +469,7 @@ function TableChunkCard({ isOriginalPreviewAvailable={isOriginalPreviewAvailable} onChunkClick={onChunkClick} > - + {safeHtml ? (

{chunk.summary - ? renderHighlightedSearchText(chunk.summary, searchQuery) + ? chunk.summary : "Table content is not available in this view."}

)} - + ); } -function renderHighlightedSearchText( - text: string, - searchQuery: string, -): ReactNode { - const normalizedQuery = searchQuery.trim().toLocaleLowerCase(); - if (!normalizedQuery) return text; - - const normalizedText = text.toLocaleLowerCase(); - const parts: ReactNode[] = []; - let cursor = 0; - let matchIndex = normalizedText.indexOf(normalizedQuery, cursor); - - while (matchIndex >= 0) { - if (matchIndex > cursor) { - parts.push(text.slice(cursor, matchIndex)); - } - - const matchEnd = matchIndex + normalizedQuery.length; - parts.push( - - {text.slice(matchIndex, matchEnd)} - , - ); - cursor = matchEnd; - matchIndex = normalizedText.indexOf(normalizedQuery, cursor); - } - - if (cursor < text.length) { - parts.push(text.slice(cursor)); - } - - return parts.length === 1 ? parts[0] : parts; -} - function renderChunkIcon(type: ParsedChunkView["type"]): ReactNode { if (type === "image") return ; if (type === "table") return ; diff --git a/src/domains/chunks/index.ts b/src/domains/chunks/index.ts index d3d98ce..d774708 100644 --- a/src/domains/chunks/index.ts +++ b/src/domains/chunks/index.ts @@ -7,7 +7,7 @@ import type { ChatCitationView } from "@/domains/chat/types" import type { ParsedChunkView } from "@/domains/chunks/types" const documentChunkPageSize = 200 -const defaultChunkPageSize = 100 +const defaultChunkPageSize = 50 const maximumChunkPageSize = 200 export type ChunkKnowhereClient = { diff --git a/src/domains/workspace/client.ts b/src/domains/workspace/client.ts index e92d56a..1cea55a 100644 --- a/src/domains/workspace/client.ts +++ b/src/domains/workspace/client.ts @@ -16,7 +16,7 @@ const workspaceClientKeys = { } as const const workspaceClientConfig = { - sourceChunkPageSize: 100, + sourceChunkPageSize: 50, } as const type SourceChunksResponse = { From 116dae5e102384478493beaf27834bfcbebfc3d1 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 14:46:49 +0800 Subject: [PATCH 08/16] fix: style dashboard link as text link instead of button --- src/components/top-nav.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/top-nav.tsx b/src/components/top-nav.tsx index e33c20b..c0fbab5 100644 --- a/src/components/top-nav.tsx +++ b/src/components/top-nav.tsx @@ -39,7 +39,7 @@ export function TopNav({ Dashboard From cb68b3d042ab9af0ebed7766b39d919385f8c3f5 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 14:47:08 +0800 Subject: [PATCH 09/16] feat: introduce upstash/workflow --- package.json | 1 + pnpm-lock.yaml | 38 ++++++++++++++ src/app/api/sources/reconcile/route.ts | 50 ++++++++++++++++++ src/domains/sources/background-reconcile.ts | 57 +++++++++++++++++++++ src/domains/sources/reconcile.ts | 4 +- src/domains/sources/route-dependencies.ts | 2 + src/domains/sources/route-listing.ts | 33 ++++-------- src/domains/sources/route-service.test.ts | 17 +++--- src/domains/sources/route-types.ts | 1 + src/domains/sources/route-upload.ts | 8 +-- src/domains/workspace/initial-state.test.ts | 28 +++++----- src/domains/workspace/initial-state.ts | 34 ++++-------- src/infrastructure/db/schema.ts | 4 -- src/integrations/knowhere-demo.ts | 13 +++++ 14 files changed, 210 insertions(+), 80 deletions(-) create mode 100644 src/app/api/sources/reconcile/route.ts create mode 100644 src/domains/sources/background-reconcile.ts diff --git a/package.json b/package.json index bb9d756..3a13bf2 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "@radix-ui/react-tabs": "^1.1.13", "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-virtual": "^3.13.24", + "@upstash/workflow": "^1.2.1", "@vercel/blob": "^2.3.3", "ai": "^6.0.175", "class-variance-authority": "^0.7.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 303e312..16c3536 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -53,6 +53,9 @@ importers: '@tanstack/react-virtual': specifier: ^3.13.24 version: 3.13.24(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@upstash/workflow': + specifier: ^1.2.1 + version: 1.2.1(zod@4.4.3) '@vercel/blob': specifier: ^2.3.3 version: 2.3.3 @@ -2311,6 +2314,14 @@ packages: cpu: [x64] os: [win32] + '@upstash/qstash@2.11.0': + resolution: {integrity: sha512-AfPPxsUeOJCrxMQ9dkh1RZL40wgxCsUNFkxrbBSomC3U4j4qKFRawU8sDK+dqqH+sZFUS1biVM4GK46d5Tg2Vg==} + + '@upstash/workflow@1.2.1': + resolution: {integrity: sha512-G2WfWruKXbPpKJNyCWS097jtRZXW993BeMPYJPc7gGoEjnpJTH9S9j/YsfRJXZxWBPAuHB31tMzmKMGWAkAMQw==} + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + '@vercel/blob@2.3.3': resolution: {integrity: sha512-MtD7VLo6hU07eHR7bmk5SIMD290q574UaNYTe46qeyRT+hWrCy26CoAqfd7PnIefVXvRehRZBzukxuTO9iGTVg==} engines: {node: '>=20.0.0'} @@ -2705,6 +2716,9 @@ packages: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} + crypto-js@4.2.0: + resolution: {integrity: sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==} + css-tree@3.2.1: resolution: {integrity: sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==} engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0} @@ -3750,6 +3764,9 @@ packages: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true + jose@5.10.0: + resolution: {integrity: sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==} + jose@6.2.3: resolution: {integrity: sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==} @@ -4211,6 +4228,10 @@ packages: resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} engines: {node: '>= 0.6'} + neverthrow@7.2.0: + resolution: {integrity: sha512-iGBUfFB7yPczHHtA8dksKTJ9E8TESNTAx1UQWW6TzMF280vo9jdPYpLUXrMN1BCkPdHFdNG3fxOt2CUad8KhAw==} + engines: {node: '>=18'} + next-themes@0.4.6: resolution: {integrity: sha512-pZvgD5L0IEvX5/9GWyHMf3m8BKiVQwsCMHfoFosXtXBMnaS0ZnIJ9ST4b4NqLVKDEm8QBxoNNGNaBv2JNF6XNA==} peerDependencies: @@ -7104,6 +7125,17 @@ snapshots: '@unrs/resolver-binding-win32-x64-msvc@1.11.1': optional: true + '@upstash/qstash@2.11.0': + dependencies: + crypto-js: 4.2.0 + jose: 5.10.0 + neverthrow: 7.2.0 + + '@upstash/workflow@1.2.1(zod@4.4.3)': + dependencies: + '@upstash/qstash': 2.11.0 + zod: 4.4.3 + '@vercel/blob@2.3.3': dependencies: async-retry: 1.3.3 @@ -7508,6 +7540,8 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 + crypto-js@4.2.0: {} + css-tree@3.2.1: dependencies: mdn-data: 2.27.1 @@ -8671,6 +8705,8 @@ snapshots: jiti@2.7.0: {} + jose@5.10.0: {} + jose@6.2.3: {} js-tokens@4.0.0: {} @@ -9325,6 +9361,8 @@ snapshots: negotiator@1.0.0: {} + neverthrow@7.2.0: {} + next-themes@0.4.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4): dependencies: react: 19.2.4 diff --git a/src/app/api/sources/reconcile/route.ts b/src/app/api/sources/reconcile/route.ts new file mode 100644 index 0000000..c28bfa4 --- /dev/null +++ b/src/app/api/sources/reconcile/route.ts @@ -0,0 +1,50 @@ +import { serve } from "@upstash/workflow/nextjs" + +import { reconcileSourcesForWorkspace } from "@/domains/sources/reconcile" +import { makeKnowhereClient } from "@/integrations/knowhere" +import { logger } from "@/lib/logger" + +type ReconcilePayload = { + readonly workspaceId: string + readonly sourceId: string + readonly apiKey: string +} + +const MAX_POLL_ATTEMPTS = 60 +const INITIAL_DELAY_S = 3 +const MAX_DELAY_S = 30 + +export const { POST } = serve(async (context) => { + const { workspaceId, sourceId, apiKey } = context.requestPayload + const workspace = { id: workspaceId } + const client = makeKnowhereClient(apiKey) + let delay = INITIAL_DELAY_S + + for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt++) { + const resolved = await context.run(`poll-${attempt}`, async () => { + const sources = await reconcileSourcesForWorkspace(workspace, client) + const source = sources.find((s) => s.id === sourceId) + if (!source || source.status !== "parsing") { + return { done: true, status: source?.status ?? "gone" } as const + } + return { done: false } as const + }) + + if (resolved.done) { + logger.info("workflow: source resolved", { + sourceId, + status: resolved.status, + attempts: attempt + 1, + }) + return + } + + await context.sleep(`wait-${attempt}`, delay) + delay = Math.min(Math.round(delay * 1.5), MAX_DELAY_S) + } + + logger.error("workflow: exhausted poll attempts", { + sourceId, + maxAttempts: MAX_POLL_ATTEMPTS, + }) +}) diff --git a/src/domains/sources/background-reconcile.ts b/src/domains/sources/background-reconcile.ts new file mode 100644 index 0000000..c1c056c --- /dev/null +++ b/src/domains/sources/background-reconcile.ts @@ -0,0 +1,57 @@ +import "server-only" + +import { Client } from "@upstash/workflow" + +import { sourceWorkflowRuntime } from "./workflow-runtime" +import { logger } from "@/lib/logger" + +const triggeredSourceIds = new Set() + +function createClient(): Client { + return new Client({ token: process.env.QSTASH_TOKEN! }) +} + +function resolveBaseURL(): string { + if (process.env.VERCEL_URL) return `https://${process.env.VERCEL_URL}` + return process.env.NOTEBOOK_PUBLIC_URL ?? "http://localhost:3000" +} + +export async function startBackgroundReconciliation( + workspaceId: string, + sourceId: string, + apiKey: string, +): Promise { + if (triggeredSourceIds.has(sourceId)) return + triggeredSourceIds.add(sourceId) + + try { + await createClient().trigger({ + url: `${resolveBaseURL()}/api/sources/reconcile`, + body: { workspaceId, sourceId, apiKey }, + retries: 3, + }) + logger.info("background-reconcile: workflow triggered", { sourceId }) + } catch (error) { + triggeredSourceIds.delete(sourceId) + logger.error("background-reconcile: failed to trigger workflow", { + sourceId, + error: String(error), + }) + } +} + +export async function reconcileStaleSources( + workspaceId: string, + apiKey: string, +): Promise { + try { + const sources = await sourceWorkflowRuntime.listForWorkspace(workspaceId) + for (const source of sources) { + if (source.status === "parsing" && source.knowhereJobId) { + void startBackgroundReconciliation(workspaceId, source.id, apiKey) + } + } + } catch { + // Best-effort sweep; listing failures must not block the caller. + } +} diff --git a/src/domains/sources/reconcile.ts b/src/domains/sources/reconcile.ts index 3be1954..6a27fdd 100644 --- a/src/domains/sources/reconcile.ts +++ b/src/domains/sources/reconcile.ts @@ -4,7 +4,7 @@ import { del } from "@vercel/blob" import type Knowhere from "@ontos-ai/knowhere-sdk" import type { JobResult } from "@ontos-ai/knowhere-sdk" -import type { Source, Workspace } from "@/infrastructure/db/schema" +import type { Source } from "@/infrastructure/db/schema" import { storeParsedResultAssets, type StoreParsedResultAssetsInput, @@ -30,7 +30,7 @@ type SourceReconcileDependencies = { } export async function reconcileSourcesForWorkspace( - workspace: Workspace, + workspace: { readonly id: string }, client: Knowhere, deps: SourceReconcileDependencies = {}, ): Promise { diff --git a/src/domains/sources/route-dependencies.ts b/src/domains/sources/route-dependencies.ts index 54bf0c2..0af5abb 100644 --- a/src/domains/sources/route-dependencies.ts +++ b/src/domains/sources/route-dependencies.ts @@ -13,6 +13,7 @@ import { getCurrentUser, requireUser } from "@/infrastructure/auth" import { workspaceService } from "@/domains/workspace/service" import { sourceViewOptionsBySourceId as getDefaultSourceViewOptionsBySourceId } from "./counts" import { reconcileSourcesForWorkspace as reconcileDefaultSourcesForWorkspace } from "./reconcile" +import { sourceWorkflowRuntime } from "./workflow-runtime" import { sourceService as defaultSourceService } from "./service" import type { SourceRouteKnowhereClient, @@ -35,6 +36,7 @@ const defaultDependencies: SourceRouteServiceDependencies = { loadChunksForSource, makeKnowhereClient: (apiKey: string) => makeDefaultKnowhereClient(apiKey) as SourceRouteKnowhereClient, + listSourcesForWorkspace: sourceWorkflowRuntime.listForWorkspace, reconcileSourcesForWorkspace: (workspace, client) => reconcileDefaultSourcesForWorkspace( workspace, diff --git a/src/domains/sources/route-listing.ts b/src/domains/sources/route-listing.ts index 1255136..434dd3a 100644 --- a/src/domains/sources/route-listing.ts +++ b/src/domains/sources/route-listing.ts @@ -7,9 +7,9 @@ import { resolveWorkspaceDemoSources, } from "@/domains/demo/workspace-source-resolution" import { routeResult } from "@/lib/route-result" -import type { DemoCatalog } from "@/integrations/knowhere-demo" +import { knowhereDemoApi } from "@/integrations/knowhere-demo" import { toSourceView } from "./view" -import { getClientForWorkspace } from "./route-dependencies" +import { reconcileStaleSources } from "./background-reconcile" import type { JsonRouteResult, ListSourcesBody, @@ -23,8 +23,8 @@ type RouteListingDependencies = Pick< | "ensureWorkspace" | "getCurrentUser" | "getSourceViewOptionsBySourceId" + | "listSourcesForWorkspace" | "makeKnowhereClient" - | "reconcileSourcesForWorkspace" > & { readonly demoApi: Pick readonly sourceService: Pick< @@ -39,8 +39,6 @@ type RouteListing = { ) => Promise> } -const emptyDemoCatalog: DemoCatalog = { sources: [] } - function createRouteListing(deps: RouteListingDependencies): RouteListing { return { listSources: (input: ListSourcesInput) => listSources(input, deps), @@ -57,14 +55,9 @@ async function listSources( return routeResult.ok({ sources: catalog.sources.map(demoView.toSourceView) }) } - const catalog = await fetchOptionalDemoCatalog(deps.demoApi.fetchCatalog) + const catalog = await knowhereDemoApi.fetchOptionalCatalog(deps.demoApi.fetchCatalog) const workspace = await deps.ensureWorkspace(user.id) - const client = await getClientForWorkspace( - workspace.id, - input.cookieHeader, - deps, - ) - const sources = await deps.reconcileSourcesForWorkspace(workspace, client) + const sources = await deps.listSourcesForWorkspace(workspace.id) const demoSourceResolution = resolveWorkspaceDemoSources(sources, catalog) const sourcesNeedingKnowhereChunkCount = getWorkspaceSourcesNeedingKnowhereChunkCount( @@ -75,6 +68,12 @@ async function listSources( demoSourceResolution.workspaceSources, catalog, ) + const apiKey = await deps.ensureApiKeyForWorkspace( + workspace.id, + input.cookieHeader, + ) + const client = deps.makeKnowhereClient(apiKey) + void reconcileStaleSources(workspace.id, apiKey) const sourceOptions = await Effect.runPromise( deps.getSourceViewOptionsBySourceId( sourcesNeedingKnowhereChunkCount, @@ -106,14 +105,4 @@ async function listSources( }) } -async function fetchOptionalDemoCatalog( - fetchDemoCatalog: () => Promise, -): Promise { - try { - return await fetchDemoCatalog() - } catch { - return emptyDemoCatalog - } -} - export { createRouteListing } diff --git a/src/domains/sources/route-service.test.ts b/src/domains/sources/route-service.test.ts index 7276d29..650fce1 100644 --- a/src/domains/sources/route-service.test.ts +++ b/src/domains/sources/route-service.test.ts @@ -58,7 +58,7 @@ describe("source route service", () => { const getSourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map([[source.id, { chunkCount: 8 }]])), ); - const reconcileSourcesForWorkspace = vi.fn(async () => [source]); + const listSourcesForWorkspace = vi.fn(async () => [source]); const listHiddenDemoSourceIds = vi.fn(async () => []); const listing = createRouteListing({ demoApi: { @@ -73,7 +73,7 @@ describe("source route service", () => { })), getSourceViewOptionsBySourceId, makeKnowhereClient: vi.fn(() => knowhereClient), - reconcileSourcesForWorkspace, + listSourcesForWorkspace, sourceService: { listHiddenDemoSourceIds }, }); @@ -99,10 +99,7 @@ describe("source route service", () => { workspace.id, "session=abc", ); - expect(reconcileSourcesForWorkspace).toHaveBeenCalledWith( - workspace, - knowhereClient, - ); + expect(listSourcesForWorkspace).toHaveBeenCalledWith(workspace.id); expect(listHiddenDemoSourceIds).toHaveBeenCalledWith(workspace.id); }); @@ -151,7 +148,7 @@ describe("source route service", () => { })), getSourceViewOptionsBySourceId, makeKnowhereClient: vi.fn(() => knowhereClient), - reconcileSourcesForWorkspace: vi.fn(async () => [legacyFakeSource, source]), + listSourcesForWorkspace: vi.fn(async () => [legacyFakeSource, source]), sourceService: { listHiddenDemoSourceIds: vi.fn(async () => []) }, }); @@ -220,7 +217,7 @@ describe("source route service", () => { })), getSourceViewOptionsBySourceId, makeKnowhereClient: vi.fn(() => knowhereClient), - reconcileSourcesForWorkspace: vi.fn(async () => [legacyFakeSource]), + listSourcesForWorkspace: vi.fn(async () => [legacyFakeSource]), sourceService: { listHiddenDemoSourceIds: vi.fn(async () => []) }, }); @@ -296,7 +293,7 @@ describe("source route service", () => { })), getSourceViewOptionsBySourceId, makeKnowhereClient: vi.fn(() => knowhereClient), - reconcileSourcesForWorkspace: vi.fn(async () => [nonReadyLegacySource]), + listSourcesForWorkspace: vi.fn(async () => [nonReadyLegacySource]), sourceService: { listHiddenDemoSourceIds: vi.fn(async () => []) }, }); @@ -363,7 +360,7 @@ describe("source route service", () => { })), getSourceViewOptionsBySourceId, makeKnowhereClient: vi.fn(() => knowhereClient), - reconcileSourcesForWorkspace: vi.fn(async () => [materializedSource]), + listSourcesForWorkspace: vi.fn(async () => [materializedSource]), sourceService: { listHiddenDemoSourceIds: vi.fn(async () => []) }, }); diff --git a/src/domains/sources/route-types.ts b/src/domains/sources/route-types.ts index e1e831d..f6e9dad 100644 --- a/src/domains/sources/route-types.ts +++ b/src/domains/sources/route-types.ts @@ -174,6 +174,7 @@ type SourceRouteServiceDependencies = { readonly loadChunkPageForSource: typeof loadChunkPageForSource readonly loadChunksForSource: typeof loadChunksForSource readonly makeKnowhereClient: (apiKey: string) => SourceRouteKnowhereClient + readonly listSourcesForWorkspace: (workspaceId: string) => Promise readonly reconcileSourcesForWorkspace: ( workspace: Workspace, client: SourceRouteKnowhereClient, diff --git a/src/domains/sources/route-upload.ts b/src/domains/sources/route-upload.ts index 6150f47..f8404e0 100644 --- a/src/domains/sources/route-upload.ts +++ b/src/domains/sources/route-upload.ts @@ -1,7 +1,7 @@ import type { Source, Workspace } from "@/infrastructure/db/schema" import { routeResult } from "@/lib/route-result" +import { startBackgroundReconciliation } from "./background-reconcile" import { validateSourceBlobUploadInput } from "./blob-upload" -import { getClientForWorkspace } from "./route-dependencies" import type { JsonRouteResult, SourceRouteKnowhereClient, @@ -56,16 +56,18 @@ async function uploadSource( } const workspace = await deps.ensureWorkspace(user.id) - const client = await getClientForWorkspace( + const apiKey = await deps.ensureApiKeyForWorkspace( workspace.id, input.cookieHeader, - deps, ) + const client = deps.makeKnowhereClient(apiKey) const source = await uploadToKnowhere(workspace, input.upload, client, deps) .finally(() => { input.onUploadFinished?.() }) + startBackgroundReconciliation(workspace.id, source.id, apiKey) + return routeResult.ok({ source: toSourceView(source) }, 201) } diff --git a/src/domains/workspace/initial-state.test.ts b/src/domains/workspace/initial-state.test.ts index 47f48a7..322f40c 100644 --- a/src/domains/workspace/initial-state.test.ts +++ b/src/domains/workspace/initial-state.test.ts @@ -81,7 +81,7 @@ describe("loadWorkspaceShellInitialState", () => { }, ]) expect(state.loginUrl).toBe("/login") - expect(deps.reconcileSourcesForWorkspace).not.toHaveBeenCalled() + expect(deps.listSourcesForWorkspace).not.toHaveBeenCalled() }) it("exposes the configured Dashboard origin to the shell", async () => { @@ -98,7 +98,7 @@ describe("loadWorkspaceShellInitialState", () => { const thread = makeThread(workspace.id) const deps = createDependencies({ listChatThreads: vi.fn(async () => [thread]), - reconcileSourcesForWorkspace: vi.fn(async () => [source]), + listSourcesForWorkspace: vi.fn(async () => [source]), sourceViewOptionsBySourceId: vi.fn(() => Effect.succeed(new Map([[source.id, { chunkCount: 2 }]])), ), @@ -143,7 +143,7 @@ describe("loadWorkspaceShellInitialState", () => { fetchDemoCatalog: vi.fn(async () => { throw new Error("Demo API unavailable.") }), - reconcileSourcesForWorkspace: vi.fn(async () => [legacyFakeSource, source]), + listSourcesForWorkspace: vi.fn(async () => [legacyFakeSource, source]), sourceViewOptionsBySourceId, }) @@ -177,7 +177,7 @@ describe("loadWorkspaceShellInitialState", () => { const sourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map())) const deps = createDependencies({ listHiddenDemoSourceIds: vi.fn(async () => ["another-demo"]), - reconcileSourcesForWorkspace: vi.fn(async () => [materializedSource]), + listSourcesForWorkspace: vi.fn(async () => [materializedSource]), sourceViewOptionsBySourceId, }) @@ -205,7 +205,7 @@ describe("loadWorkspaceShellInitialState", () => { }) const sourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map())) const deps = createDependencies({ - reconcileSourcesForWorkspace: vi.fn(async () => [legacyFakeSource]), + listSourcesForWorkspace: vi.fn(async () => [legacyFakeSource]), sourceViewOptionsBySourceId, }) @@ -233,7 +233,7 @@ describe("loadWorkspaceShellInitialState", () => { }) const sourceViewOptionsBySourceId = vi.fn(() => Effect.succeed(new Map())) const deps = createDependencies({ - reconcileSourcesForWorkspace: vi.fn(async () => [nonReadyLegacySource]), + listSourcesForWorkspace: vi.fn(async () => [nonReadyLegacySource]), sourceViewOptionsBySourceId, }) @@ -322,13 +322,13 @@ describe("loadWorkspaceShellInitialState", () => { ]) }) - it("reconciles source state during authenticated shell load", async () => { + it("lists workspace sources without blocking on reconciliation", async () => { const workspace = makeWorkspace() const readySource = makeSource(workspace.id, { status: "ready", knowhereDocumentId: "document_1", }) - const reconcileSourcesForWorkspace = vi.fn(async () => [readySource]) + const listSourcesForWorkspace = vi.fn(async () => [readySource]) const deps = { ...createDependencies({ getOptionalAuthenticated: vi.fn(async () => ({ @@ -340,15 +340,12 @@ describe("loadWorkspaceShellInitialState", () => { workspace, })), }), - reconcileSourcesForWorkspace, + listSourcesForWorkspace, } satisfies InitialStateDependencies const state = await loadWorkspaceShellInitialState(deps) - expect(reconcileSourcesForWorkspace).toHaveBeenCalledWith( - workspace, - expect.any(Object), - ) + expect(listSourcesForWorkspace).toHaveBeenCalledWith(workspace.id) expect(state.sources).toEqual([ expect.objectContaining({ id: "demo-tsla-q4-2025", @@ -379,14 +376,14 @@ function createDependencies( return { fetchDemoCatalog: vi.fn(async () => makeDemoCatalog()), - getClientForWorkspace: vi.fn(async () => ({ client })), + getClientForWorkspace: vi.fn(async () => ({ client, apiKey: "sk_test" })), getGuest: vi.fn(async () => ({ loginUrl: "/login" })), getOptionalAuthenticated: vi.fn(async () => ({ user, workspace })), ensureDemoChatThread: vi.fn(async () => null), listChatThreads: vi.fn(async () => []), listHiddenDemoSourceIds: vi.fn(async () => []), listMessages: vi.fn(async () => []), - reconcileSourcesForWorkspace: vi.fn(async () => []), + listSourcesForWorkspace: vi.fn(async () => []), sourceViewOptionsBySourceId: vi.fn(() => Effect.succeed(new Map())), ...overrides, } @@ -482,6 +479,7 @@ function makeThread( title: "Revenue", createdAt: new Date("2026-05-10T00:00:00.000Z"), updatedAt: new Date("2026-05-10T00:00:00.000Z"), + deletedAt: null, ...overrides, } diff --git a/src/domains/workspace/initial-state.ts b/src/domains/workspace/initial-state.ts index 574006a..c5f0843 100644 --- a/src/domains/workspace/initial-state.ts +++ b/src/domains/workspace/initial-state.ts @@ -12,8 +12,9 @@ import { import { chatThreadService } from "@/domains/chat/thread-service" import { toChatMessageView, toChatThreadView } from "@/domains/chat/view" import { sourceViewOptionsBySourceId as getSourceViewOptionsBySourceId } from "@/domains/sources/counts" -import { reconcileSourcesForWorkspace as reconcileDefaultSourcesForWorkspace } from "@/domains/sources/reconcile" import { sourceService } from "@/domains/sources/service" +import { sourceWorkflowRuntime } from "@/domains/sources/workflow-runtime" +import { reconcileStaleSources } from "@/domains/sources/background-reconcile" import type { SourceView } from "@/domains/sources/types" import { toSourceView } from "@/domains/sources/view" import type { AuthUser } from "@/infrastructure/auth" @@ -46,14 +47,13 @@ type WorkspaceShellInitialState = { } type WorkspaceShellInitialStateClient = - Parameters[1] & - Parameters[1] + Parameters[1] type WorkspaceShellInitialStateDependencies = { readonly fetchDemoCatalog: () => Promise readonly getClientForWorkspace: ( workspace: Workspace, - ) => Promise<{ readonly client: WorkspaceShellInitialStateClient }> + ) => Promise<{ readonly apiKey: string; readonly client: WorkspaceShellInitialStateClient }> readonly getGuest: () => Promise<{ readonly loginUrl: string }> readonly getOptionalAuthenticated: () => Promise<{ readonly user: AuthUser @@ -72,10 +72,7 @@ type WorkspaceShellInitialStateDependencies = { workspaceId: string, threadId: string, ) => Promise - readonly reconcileSourcesForWorkspace: ( - workspace: Workspace, - client: WorkspaceShellInitialStateClient, - ) => Promise + readonly listSourcesForWorkspace: (workspaceId: string) => Promise readonly sourceViewOptionsBySourceId: ( sources: readonly Source[], client: WorkspaceShellInitialStateClient, @@ -91,12 +88,10 @@ const defaultDependencies: WorkspaceShellInitialStateDependencies = { listChatThreads: chatThreadService.listForWorkspace, listHiddenDemoSourceIds: sourceService.listHiddenDemoSourceIds, listMessages: chatThreadService.listMessages, - reconcileSourcesForWorkspace: reconcileDefaultSourcesForWorkspace, + listSourcesForWorkspace: sourceWorkflowRuntime.listForWorkspace, sourceViewOptionsBySourceId: getSourceViewOptionsBySourceId, } -const emptyDemoCatalog: DemoCatalog = { sources: [] } - export async function loadWorkspaceShellInitialState( deps: WorkspaceShellInitialStateDependencies = defaultDependencies, ): Promise { @@ -115,9 +110,8 @@ export async function loadWorkspaceShellInitialState( } const { user, workspace } = context - const demoCatalog = await fetchOptionalDemoCatalog(deps.fetchDemoCatalog) - const { client } = await deps.getClientForWorkspace(workspace) - const sources = await deps.reconcileSourcesForWorkspace(workspace, client) + const demoCatalog = await knowhereDemoApi.fetchOptionalCatalog(deps.fetchDemoCatalog) + const sources = await deps.listSourcesForWorkspace(workspace.id) const demoSourceResolution = resolveWorkspaceDemoSources( sources, demoCatalog, @@ -160,6 +154,8 @@ export async function loadWorkspaceShellInitialState( demoSourceResolution.workspaceSources, demoCatalog, ) + const { client, apiKey } = await deps.getClientForWorkspace(workspace) + void reconcileStaleSources(workspace.id, apiKey) const sourceOptions = await Effect.runPromise( deps.sourceViewOptionsBySourceId(sourcesNeedingKnowhereChunkCount, client), ) @@ -194,13 +190,3 @@ export async function loadWorkspaceShellInitialState( function resolveDashboardUrl(): string | undefined { return process.env.DASHBOARD_ORIGIN } - -async function fetchOptionalDemoCatalog( - fetchDemoCatalog: () => Promise, -): Promise { - try { - return await fetchDemoCatalog() - } catch { - return emptyDemoCatalog - } -} diff --git a/src/infrastructure/db/schema.ts b/src/infrastructure/db/schema.ts index 4845c44..76f6e2c 100644 --- a/src/infrastructure/db/schema.ts +++ b/src/infrastructure/db/schema.ts @@ -111,13 +111,9 @@ export const sources = pgTable( deletedAt: timestamp("deleted_at", { withTimezone: true }), }, (t) => [ - // Sidebar list query: per workspace, newest first, soft-deleted - // rows hidden. Partial index keeps the hot path lean. index("sources_workspace_created_idx") .on(t.workspaceId, t.createdAt.desc()) .where(sql`deleted_at IS NULL`), - // Reconcile sweep picks up anything still in `uploading` or - // `parsing`. Small cardinality, small index. index("sources_workspace_status_idx").on(t.workspaceId, t.status), uniqueIndex("sources_workspace_demo_key_idx").on(t.workspaceId, t.demoKey), ], diff --git a/src/integrations/knowhere-demo.ts b/src/integrations/knowhere-demo.ts index 2f5a7dd..87f2dc9 100644 --- a/src/integrations/knowhere-demo.ts +++ b/src/integrations/knowhere-demo.ts @@ -175,8 +175,21 @@ type MaterializedDemoSourceResponse = { const DEFAULT_KNOWHERE_BASE_URL = "https://api.knowhereto.ai" +const emptyCatalog: DemoCatalog = { sources: [] } + +async function fetchOptionalCatalog( + fetcher?: () => Promise, +): Promise { + try { + return await (fetcher ?? fetchCatalog)() + } catch { + return emptyCatalog + } +} + export const knowhereDemoApi = { fetchCatalog, + fetchOptionalCatalog, fetchChunkPage, materializeSources, resolveApiURL, From dfa955683e4113783c8649cfb7178465ddf539ac Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 15:37:52 +0800 Subject: [PATCH 10/16] refactor: convert server-side code to Effect patterns Replace raw fetch/async/try-catch/throw with Effect.gen, Effect.fn, Effect.tryPromise, and structured error handling across foundations, domain logic, route services, and API handlers. --- src/app/api/demo-sources/materialize/route.ts | 133 ++++++----- src/app/api/source-uploads/blob/route.ts | 157 +++++++------ src/domains/chat/chat-turn-persistence.ts | 14 +- src/domains/chat/route-answer.ts | 78 ++++--- src/domains/chat/route-threads.ts | 121 ++++++---- src/domains/sources/background-reconcile.ts | 89 ++++--- src/domains/sources/lifecycle.ts | 136 +++++++---- src/domains/sources/parsed-result-assets.ts | 117 +++++++--- src/domains/sources/reconcile.ts | 72 ++++-- src/domains/sources/route-archive.ts | 91 ++++---- src/domains/sources/route-chunks.ts | 142 +++++++----- src/domains/sources/route-listing.ts | 120 ++++++---- src/domains/sources/route-upload.ts | 101 ++++---- src/domains/workspace/initial-state.ts | 217 +++++++++++------- src/domains/workspace/request-context.ts | 111 ++++++--- src/integrations/knowhere-demo.ts | 196 +++++++++++----- src/lib/api-error-response.ts | 26 ++- src/lib/route-result.ts | 43 +++- 18 files changed, 1257 insertions(+), 707 deletions(-) diff --git a/src/app/api/demo-sources/materialize/route.ts b/src/app/api/demo-sources/materialize/route.ts index 273e82c..7304ada 100644 --- a/src/app/api/demo-sources/materialize/route.ts +++ b/src/app/api/demo-sources/materialize/route.ts @@ -1,3 +1,4 @@ +import { Effect } from "effect" import type { NextResponse } from "next/server" import { sourceService } from "@/domains/sources/service" @@ -8,72 +9,94 @@ import { nextRouteResponse } from "@/lib/next-route-response" import { routeResult } from "@/lib/route-result" export async function POST(request: Request): Promise { - const body = await routeResult.readJson(request) - if (!body.ok) { - return nextRouteResponse.toNextResponse( - routeResult.badRequest("Invalid request body."), - ) - } + return Effect.runPromise( + Effect.gen(function* () { + const body = yield* Effect.tryPromise(() => + routeResult.readJson(request), + ) + if (!body.ok) { + return nextRouteResponse.toNextResponse( + routeResult.badRequest("Invalid request body."), + ) + } - const demoSourceIds = getDemoSourceIds(body.value) - if (demoSourceIds.length === 0) { - return nextRouteResponse.toNextResponse( - routeResult.badRequest("Select at least one demo source."), - ) - } + const demoSourceIds = getDemoSourceIds(body.value) + if (demoSourceIds.length === 0) { + return nextRouteResponse.toNextResponse( + routeResult.badRequest("Select at least one demo source."), + ) + } - try { - const { apiKey, workspace } = - await notebookRequestContext.getAuthenticatedWithClient() - const hiddenDemoSourceIds = new Set( - await sourceService.listHiddenDemoSourceIds(workspace.id), - ) - const visibleDemoSourceIds = demoSourceIds.filter( - (demoSourceId) => !hiddenDemoSourceIds.has(demoSourceId), - ) - if (visibleDemoSourceIds.length === 0) { - return nextRouteResponse.toNextResponse( - routeResult.badRequest("Selected demo sources are no longer available."), + const { apiKey, workspace } = yield* Effect.tryPromise(() => + notebookRequestContext.getAuthenticatedWithClient(), ) - } - - const materializedSources = await knowhereDemoApi.materializeSources({ - apiKey, - namespace: workspace.namespace, - demoSourceIds: visibleDemoSourceIds, - }) - const sources = await Promise.all( - materializedSources.map(async (source) => { - const row = await sourceService.upsertMaterializedDemoSource( - workspace.id, - { - demoSourceId: source.demoSourceId, - title: source.title, - mimeType: source.mimeType, - sizeBytes: source.sizeBytes, - knowhereDocumentId: source.documentId, - originalBlobUrl: `/api/demo-sources/${encodeURIComponent( - source.demoSourceId, - )}/original`, - }, + const hiddenDemoSourceIds = new Set( + yield* Effect.tryPromise(() => + sourceService.listHiddenDemoSourceIds(workspace.id), + ), + ) + const visibleDemoSourceIds = demoSourceIds.filter( + (demoSourceId) => !hiddenDemoSourceIds.has(demoSourceId), + ) + if (visibleDemoSourceIds.length === 0) { + return nextRouteResponse.toNextResponse( + routeResult.badRequest( + "Selected demo sources are no longer available.", + ), ) - return toSourceView(row, { chunkCount: source.chunkCount }) - }), - ) + } - return nextRouteResponse.toNextResponse(routeResult.ok({ sources })) - } catch { - return nextRouteResponse.toNextResponse( - routeResult.error(502, "Demo sources could not be prepared right now."), - ) - } + const materializedSources = yield* Effect.tryPromise(() => + knowhereDemoApi.materializeSources({ + apiKey, + namespace: workspace.namespace, + demoSourceIds: visibleDemoSourceIds, + }), + ) + + const sources = yield* Effect.all( + materializedSources.map((source) => + Effect.gen(function* () { + const row = yield* Effect.tryPromise(() => + sourceService.upsertMaterializedDemoSource(workspace.id, { + demoSourceId: source.demoSourceId, + title: source.title, + mimeType: source.mimeType, + sizeBytes: source.sizeBytes, + knowhereDocumentId: source.documentId, + originalBlobUrl: `/api/demo-sources/${encodeURIComponent( + source.demoSourceId, + )}/original`, + }), + ) + return toSourceView(row, { chunkCount: source.chunkCount }) + }), + ), + { concurrency: "unbounded" }, + ) + + return nextRouteResponse.toNextResponse(routeResult.ok({ sources })) + }).pipe( + Effect.catchAll(() => + Effect.succeed( + nextRouteResponse.toNextResponse( + routeResult.error( + 502, + "Demo sources could not be prepared right now.", + ), + ), + ), + ), + ), + ) } function getDemoSourceIds(value: unknown): string[] { if (!isRecord(value) || !Array.isArray(value.demoSourceIds)) return [] const selectedIds = value.demoSourceIds.filter( - (item): item is string => typeof item === "string" && item.trim().length > 0, + (item): item is string => + typeof item === "string" && item.trim().length > 0, ) return Array.from(new Set(selectedIds.map((item) => item.trim()))) } diff --git a/src/app/api/source-uploads/blob/route.ts b/src/app/api/source-uploads/blob/route.ts index 2dd5c73..a2c1d92 100644 --- a/src/app/api/source-uploads/blob/route.ts +++ b/src/app/api/source-uploads/blob/route.ts @@ -1,3 +1,4 @@ +import { Effect } from "effect" import { del } from "@vercel/blob" import { handleUpload, type HandleUploadBody } from "@vercel/blob/client" import type { NextRequest, NextResponse } from "next/server" @@ -13,81 +14,107 @@ import { nextRouteResponse } from "@/lib/next-route-response" import { routeResult } from "@/lib/route-result" export async function POST(request: NextRequest): Promise { - const user = await getCurrentUser() - if (!user) { - return nextRouteResponse.toNextResponse( - routeResult.error(401, "Please log in to upload documents."), - ) - } + return Effect.runPromise( + Effect.gen(function* () { + const user = yield* Effect.tryPromise(() => getCurrentUser()) + if (!user) { + return nextRouteResponse.toNextResponse( + routeResult.error(401, "Please log in to upload documents."), + ) + } - try { - const body = (await request.json()) as HandleUploadBody - const response = await handleUpload({ - body, - request, - onBeforeGenerateToken: async (pathname, clientPayload) => { - const input = parseSourceBlobClientPayload(clientPayload) - if (!input) { - throw new Error("Invalid upload metadata.") - } + const body = (yield* Effect.tryPromise(() => + request.json(), + )) as HandleUploadBody - const validation = validateSourceBlobUploadMetadata({ - ...input, - pathname, - }) - if (!validation.ok) { - throw new Error(validation.message) - } + const result = yield* Effect.tryPromise(() => + handleUpload({ + body, + request, + onBeforeGenerateToken: async (pathname, clientPayload) => { + const input = parseSourceBlobClientPayload(clientPayload) + if (!input) { + throw new Error("Invalid upload metadata.") + } - return { - addRandomSuffix: true, - allowOverwrite: false, - maximumSizeInBytes: MAX_UPLOAD_BYTES, - tokenPayload: JSON.stringify({ - userId: user.id, - fileName: validation.title, - mimeType: validation.mimeType, - sizeBytes: input.sizeBytes, - }), - } - }, - }) + const validation = validateSourceBlobUploadMetadata({ + ...input, + pathname, + }) + if (!validation.ok) { + throw new Error(validation.message) + } - return nextRouteResponse.toNextResponse(routeResult.ok(response)) - } catch (error) { - const message = error instanceof Error - ? error.message - : "Could not prepare the upload." - return nextRouteResponse.toNextResponse(routeResult.badRequest(message)) - } + return { + addRandomSuffix: true, + allowOverwrite: false, + maximumSizeInBytes: MAX_UPLOAD_BYTES, + tokenPayload: JSON.stringify({ + userId: user.id, + fileName: validation.title, + mimeType: validation.mimeType, + sizeBytes: input.sizeBytes, + }), + } + }, + }), + ).pipe( + Effect.map((response) => + nextRouteResponse.toNextResponse(routeResult.ok(response)), + ), + Effect.catchAll((error) => { + const message = + error instanceof Error + ? error.message + : "Could not prepare the upload." + return Effect.succeed( + nextRouteResponse.toNextResponse(routeResult.badRequest(message)), + ) + }), + ) + + return result + }), + ) } export async function DELETE(request: NextRequest): Promise { - const user = await getCurrentUser() - if (!user) { - return nextRouteResponse.toNextResponse( - routeResult.error(401, "Please log in to upload documents."), - ) - } + return Effect.runPromise( + Effect.gen(function* () { + const user = yield* Effect.tryPromise(() => getCurrentUser()) + if (!user) { + return nextRouteResponse.toNextResponse( + routeResult.error(401, "Please log in to upload documents."), + ) + } - try { - const body = (await request.json()) as unknown - const pathname = getCleanupPathname(body) - if (!pathname || !isValidSourceBlobPathname(pathname)) { - return nextRouteResponse.toNextResponse( - routeResult.badRequest( - "Invalid upload path. Choose the document again.", + const body = (yield* Effect.tryPromise(() => + request.json(), + ).pipe( + Effect.catchAll( + (): Effect.Effect => Effect.succeed(null), ), - ) - } + )) as unknown + + const pathname = getCleanupPathname(body) + if (!pathname || !isValidSourceBlobPathname(pathname)) { + return nextRouteResponse.toNextResponse( + routeResult.badRequest("Invalid upload path. Choose the document again."), + ) + } - await del(pathname) - return nextRouteResponse.toNextResponse(routeResult.ok({ ok: true })) - } catch { - return nextRouteResponse.toNextResponse( - routeResult.error(500, "Could not clean up the upload."), - ) - } + yield* Effect.tryPromise(() => del(pathname)) + return nextRouteResponse.toNextResponse(routeResult.ok({ ok: true })) + }).pipe( + Effect.catchAll(() => + Effect.succeed( + nextRouteResponse.toNextResponse( + routeResult.error(500, "Could not clean up the upload."), + ), + ), + ), + ), + ) } function getCleanupPathname(body: unknown): string | null { diff --git a/src/domains/chat/chat-turn-persistence.ts b/src/domains/chat/chat-turn-persistence.ts index c2852dc..fbb759c 100644 --- a/src/domains/chat/chat-turn-persistence.ts +++ b/src/domains/chat/chat-turn-persistence.ts @@ -44,13 +44,19 @@ function createRepository( adapter: ChatThreadPersistenceAdapter = chatThreadService, ): ChatRepository { return { - ensureDefaultChatThread: adapter.ensureDefault, - findChatThreadInWorkspace: adapter.findInWorkspace, - listMessagesForThread: async (workspaceId: string, threadId: string) => { + ensureDefaultChatThread: (workspaceId) => + adapter.ensureDefault(workspaceId), + + findChatThreadInWorkspace: (workspaceId, threadId) => + adapter.findInWorkspace(workspaceId, threadId), + + listMessagesForThread: async (workspaceId, threadId) => { const messages = await adapter.listMessages(workspaceId, threadId) return messages ? [...messages] : null }, - appendMessageToThread: adapter.appendMessage, + + appendMessageToThread: (workspaceId, input) => + adapter.appendMessage(workspaceId, input), } } diff --git a/src/domains/chat/route-answer.ts b/src/domains/chat/route-answer.ts index 896c026..a7599d8 100644 --- a/src/domains/chat/route-answer.ts +++ b/src/domains/chat/route-answer.ts @@ -1,4 +1,4 @@ -import { Either } from "effect" +import { Effect, Either } from "effect" import { generateContextualRetrievalQuery, @@ -30,42 +30,62 @@ type ChatAnswerRouteService = { ) => Promise> } -async function answerChat( - input: AnswerChatInput, -): Promise> { - const body = parseChatRequestBody(input.body) - if (!body.ok) { - return routeResult.error(body.status, body.message) - } +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- - const { workspace, client } = - await notebookRequestContext.getAuthenticatedWithClient() - const sources = await reconcileSourcesForWorkspace(workspace, client) +const answerChatEffect = (input: AnswerChatInput) => + Effect.gen(function* () { + const body = parseChatRequestBody(input.body) + if (!body.ok) { + return routeResult.error(body.status, body.message) + } - try { - const result = await handleChatTurn({ - workspace, - sources, - question: body.value.question, - threadId: body.value.threadId, - excludedSourceIds: body.value.excludedSourceIds, - retrieval: client.retrieval, - generateRetrievalQuery: generateContextualRetrievalQuery, - generateAnswer: generateGroundedAnswer, - repository: chatTurnPersistence.createRepository(), - }) + const { workspace, client } = yield* Effect.tryPromise(() => + notebookRequestContext.getAuthenticatedWithClient(), + ) + const sources = yield* Effect.tryPromise(() => + reconcileSourcesForWorkspace(workspace, client), + ) + + const result = yield* Effect.tryPromise(() => + handleChatTurn({ + workspace, + sources, + question: body.value.question, + threadId: body.value.threadId, + excludedSourceIds: body.value.excludedSourceIds, + retrieval: client.retrieval, + generateRetrievalQuery: generateContextualRetrievalQuery, + generateAnswer: generateGroundedAnswer, + repository: chatTurnPersistence.createRepository(), + }), + ).pipe( + Effect.catchAll(() => + Effect.succeed( + Either.left({ + status: 401, + message: "Your session may have expired. Please refresh the page.", + }), + ), + ), + ) return Either.match(result, { onLeft: (error): RouteResponse => routeResult.error(error.status, error.message), onRight: (value): RouteResponse => routeResult.ok(value), }) - } catch { - return routeResult.error( - 401, - "Your session may have expired. Please refresh the page.", - ) - } + }) + +// --------------------------------------------------------------------------- +// Async wrapper (backward-compatible) +// --------------------------------------------------------------------------- + +async function answerChat( + input: AnswerChatInput, +): Promise> { + return Effect.runPromise(answerChatEffect(input)) } export const chatAnswerRouteService: ChatAnswerRouteService = { diff --git a/src/domains/chat/route-threads.ts b/src/domains/chat/route-threads.ts index 796996d..3a851a6 100644 --- a/src/domains/chat/route-threads.ts +++ b/src/domains/chat/route-threads.ts @@ -1,3 +1,5 @@ +import { Effect } from "effect" + import { chatThreadService } from "@/domains/chat/thread-service" import { toChatMessageView, toChatThreadView } from "@/domains/chat/view" import { notebookRequestContext } from "@/domains/workspace/request-context" @@ -48,67 +50,102 @@ type ChatThreadRouteService = { readonly listThreads: () => Promise> } -async function listThreads(): Promise> { - const { workspace } = await notebookRequestContext.getAuthenticated() - const threads = await chatThreadService.listForWorkspace(workspace.id) +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const listThreadsEffect = Effect.gen(function* () { + const { workspace } = yield* Effect.tryPromise(() => + notebookRequestContext.getAuthenticated(), + ) + const threads = yield* Effect.tryPromise(() => + chatThreadService.listForWorkspace(workspace.id), + ) return routeResult.ok({ threads: threads.map(toChatThreadView), }) -} +}) -async function createThread(): Promise> { - const { workspace } = await notebookRequestContext.getAuthenticated() - const thread = await chatThreadService.create(workspace.id) +const createThreadEffect = Effect.gen(function* () { + const { workspace } = yield* Effect.tryPromise(() => + notebookRequestContext.getAuthenticated(), + ) + const thread = yield* Effect.tryPromise(() => + chatThreadService.create(workspace.id), + ) return routeResult.ok({ thread: toChatThreadView(thread), - messages: [], + messages: [] as unknown as [], }) +}) + +const getThreadEffect = (input: ThreadInput) => + Effect.gen(function* () { + const { workspace } = yield* Effect.tryPromise(() => + notebookRequestContext.getAuthenticated(), + ) + const thread = yield* Effect.tryPromise(() => + chatThreadService.findInWorkspace(workspace.id, input.threadId), + ) + + if (!thread) { + return routeResult.error(404, "Chat thread not found.") + } + + const messages = yield* Effect.tryPromise(() => + chatThreadService.listMessages(workspace.id, input.threadId), + ) + if (!messages) { + return routeResult.error(404, "Chat thread not found.") + } + + return routeResult.ok({ + thread: toChatThreadView(thread), + messages: messages.map((message): ChatMessageView => + toChatMessageView(message), + ), + }) + }) + +const archiveThreadEffect = (input: ArchiveThreadInput) => + Effect.gen(function* () { + const { workspace } = yield* Effect.tryPromise(() => + notebookRequestContext.getAuthenticated(), + ) + const archived = yield* Effect.tryPromise(() => + chatThreadService.softDelete(workspace.id, input.threadId), + ) + if (!archived) { + return routeResult.error(404, "Chat thread not found.") + } + + return routeResult.ok({ id: input.threadId, archived: true as const }) + }) + +// --------------------------------------------------------------------------- +// Async wrappers (backward-compatible) +// --------------------------------------------------------------------------- + +async function listThreads(): Promise> { + return Effect.runPromise(listThreadsEffect) +} + +async function createThread(): Promise> { + return Effect.runPromise(createThreadEffect) } async function getThread( input: ThreadInput, ): Promise> { - const { workspace } = await notebookRequestContext.getAuthenticated() - const thread = await chatThreadService.findInWorkspace( - workspace.id, - input.threadId, - ) - - if (!thread) { - return routeResult.error(404, "Chat thread not found.") - } - - const messages = await chatThreadService.listMessages( - workspace.id, - input.threadId, - ) - if (!messages) { - return routeResult.error(404, "Chat thread not found.") - } - - return routeResult.ok({ - thread: toChatThreadView(thread), - messages: messages.map((message): ChatMessageView => - toChatMessageView(message), - ), - }) + return Effect.runPromise(getThreadEffect(input)) } async function archiveThread( input: ArchiveThreadInput, ): Promise> { - const { workspace } = await notebookRequestContext.getAuthenticated() - const archived = await chatThreadService.softDelete( - workspace.id, - input.threadId, - ) - if (!archived) { - return routeResult.error(404, "Chat thread not found.") - } - - return routeResult.ok({ id: input.threadId, archived: true }) + return Effect.runPromise(archiveThreadEffect(input)) } export const chatThreadRouteService: ChatThreadRouteService = { diff --git a/src/domains/sources/background-reconcile.ts b/src/domains/sources/background-reconcile.ts index c1c056c..8eb98c2 100644 --- a/src/domains/sources/background-reconcile.ts +++ b/src/domains/sources/background-reconcile.ts @@ -1,5 +1,6 @@ import "server-only" +import { Effect } from "effect" import { Client } from "@upstash/workflow" import { sourceWorkflowRuntime } from "./workflow-runtime" @@ -12,46 +13,78 @@ function createClient(): Client { } function resolveBaseURL(): string { - if (process.env.VERCEL_URL) return `https://${process.env.VERCEL_URL}` return process.env.NOTEBOOK_PUBLIC_URL ?? "http://localhost:3000" } +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const startBackgroundReconciliationEffect = ( + workspaceId: string, + sourceId: string, + apiKey: string, +): Effect.Effect => + Effect.gen(function* () { + if (triggeredSourceIds.has(sourceId)) return + triggeredSourceIds.add(sourceId) + + yield* Effect.tryPromise(() => + createClient().trigger({ + url: `${resolveBaseURL()}/api/sources/reconcile`, + body: { workspaceId, sourceId, apiKey }, + retries: 3, + }), + ) + yield* Effect.logInfo( + `background-reconcile: workflow triggered for ${sourceId}`, + ) + }).pipe( + Effect.catchAll((error) => + Effect.sync(() => { + triggeredSourceIds.delete(sourceId) + logger.error("background-reconcile: failed to trigger workflow", { + sourceId, + error: String(error), + }) + }), + ), + ) + +const reconcileStaleSourcesEffect = ( + workspaceId: string, + apiKey: string, +): Effect.Effect => + Effect.gen(function* () { + const sources = yield* Effect.tryPromise(() => + sourceWorkflowRuntime.listForWorkspace(workspaceId), + ) + for (const source of sources) { + if (source.status === "parsing" && source.knowhereJobId) { + yield* Effect.fork( + startBackgroundReconciliationEffect(workspaceId, source.id, apiKey), + ) + } + } + }).pipe(Effect.catchAllCause(() => Effect.void)) + +// --------------------------------------------------------------------------- +// Async wrappers (backward-compatible) +// --------------------------------------------------------------------------- + export async function startBackgroundReconciliation( workspaceId: string, sourceId: string, apiKey: string, ): Promise { - if (triggeredSourceIds.has(sourceId)) return - triggeredSourceIds.add(sourceId) - - try { - await createClient().trigger({ - url: `${resolveBaseURL()}/api/sources/reconcile`, - body: { workspaceId, sourceId, apiKey }, - retries: 3, - }) - logger.info("background-reconcile: workflow triggered", { sourceId }) - } catch (error) { - triggeredSourceIds.delete(sourceId) - logger.error("background-reconcile: failed to trigger workflow", { - sourceId, - error: String(error), - }) - } + return Effect.runPromise( + startBackgroundReconciliationEffect(workspaceId, sourceId, apiKey), + ) } export async function reconcileStaleSources( workspaceId: string, apiKey: string, ): Promise { - try { - const sources = await sourceWorkflowRuntime.listForWorkspace(workspaceId) - for (const source of sources) { - if (source.status === "parsing" && source.knowhereJobId) { - void startBackgroundReconciliation(workspaceId, source.id, apiKey) - } - } - } catch { - // Best-effort sweep; listing failures must not block the caller. - } + return Effect.runPromise(reconcileStaleSourcesEffect(workspaceId, apiKey)) } diff --git a/src/domains/sources/lifecycle.ts b/src/domains/sources/lifecycle.ts index cbc1404..1bb4f24 100644 --- a/src/domains/sources/lifecycle.ts +++ b/src/domains/sources/lifecycle.ts @@ -1,5 +1,6 @@ import "server-only" +import { Effect } from "effect" import type { JobResult } from "@ontos-ai/knowhere-sdk" import type { Source } from "@/infrastructure/db/schema" @@ -47,6 +48,75 @@ type ApplyKnowhereJobToSourceInput = { blobStore: SourceLifecycleBlobStore } +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +export const applyKnowhereJobToSourceEffect = Effect.fn( + "applyKnowhereJobToSource", +)( + function* ({ + workspaceId, + source, + job, + client, + repository, + parsedResultStore, + blobStore, + }: ApplyKnowhereJobToSourceInput) { + if (job.isDone || job.status === "done") { + if (job.documentId) { + const stored = yield* Effect.tryPromise(() => + parsedResultStore.storeParsedResultAssets({ + workspaceId, + sourceId: source.id, + job, + client, + }), + ) + yield* Effect.tryPromise(() => + repository.saveSourceParseResult(workspaceId, source.id, stored), + ) + yield* Effect.tryPromise(() => + repository.markSourceReady(workspaceId, source.id, job.documentId!), + ) + yield* cleanupStagedBlobEffect( + workspaceId, + source, + repository, + blobStore, + ) + return + } + + yield* Effect.tryPromise(() => + repository.markSourceFailed( + workspaceId, + source.id, + "Parsing finished but no document was published.", + ), + ) + yield* cleanupStagedBlobEffect(workspaceId, source, repository, blobStore) + return + } + + if (job.isFailed || job.status === "failed") { + yield* Effect.tryPromise(() => + repository.markSourceFailed( + workspaceId, + source.id, + job.error?.message ?? "Parsing failed.", + ), + ) + yield* cleanupStagedBlobEffect(workspaceId, source, repository, blobStore) + } + }, +) + +// --------------------------------------------------------------------------- +// Async wrapper (backward-compatible) +// --------------------------------------------------------------------------- + export async function applyKnowhereJobToSource({ workspaceId, source, @@ -56,51 +126,39 @@ export async function applyKnowhereJobToSource({ parsedResultStore, blobStore, }: ApplyKnowhereJobToSourceInput): Promise { - if (job.isDone || job.status === "done") { - if (job.documentId) { - const stored = await parsedResultStore.storeParsedResultAssets({ - workspaceId, - sourceId: source.id, - job, - client, - }) - await repository.saveSourceParseResult(workspaceId, source.id, stored) - await repository.markSourceReady(workspaceId, source.id, job.documentId) - await cleanupStagedBlob(workspaceId, source, repository, blobStore) - return - } - - await repository.markSourceFailed( - workspaceId, - source.id, - "Parsing finished but no document was published.", - ) - await cleanupStagedBlob(workspaceId, source, repository, blobStore) - return - } - - if (job.isFailed || job.status === "failed") { - await repository.markSourceFailed( + return Effect.runPromise( + applyKnowhereJobToSourceEffect({ workspaceId, - source.id, - job.error?.message ?? "Parsing failed.", - ) - await cleanupStagedBlob(workspaceId, source, repository, blobStore) - } + source, + job, + client, + repository, + parsedResultStore, + blobStore, + }), + ) } -async function cleanupStagedBlob( +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function cleanupStagedBlobEffect( workspaceId: string, source: Source, repository: SourceLifecycleRepository, blobStore: SourceLifecycleBlobStore, -): Promise { - if (!source.stagedBlobPathname) return +): Effect.Effect { + if (!source.stagedBlobPathname) return Effect.void - try { - await blobStore.deleteStagedSourceBlob(source.stagedBlobPathname) - await repository.clearSourceStagedBlob(workspaceId, source.id) - } catch { - // Staged upload cleanup is best-effort; source state is already advanced. - } + return Effect.gen(function* () { + yield* Effect.tryPromise(() => + blobStore.deleteStagedSourceBlob(source.stagedBlobPathname!), + ) + yield* Effect.tryPromise(() => + repository.clearSourceStagedBlob(workspaceId, source.id), + ) + }).pipe( + Effect.catchAllCause(() => Effect.void), + ) } diff --git a/src/domains/sources/parsed-result-assets.ts b/src/domains/sources/parsed-result-assets.ts index 39ae608..6a36732 100644 --- a/src/domains/sources/parsed-result-assets.ts +++ b/src/domains/sources/parsed-result-assets.ts @@ -2,6 +2,7 @@ import "server-only" import path from "node:path" import { put } from "@vercel/blob" +import { Effect } from "effect" import type { JobResult } from "@ontos-ai/knowhere-sdk" export type StoredParsedResultAssets = { @@ -52,6 +53,73 @@ export type StoreParsedResultAssetsInput = { const parsedResultDirectoryName = "parsed-result" +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +export const storeParsedResultAssetsEffect = Effect.fn( + "storeParsedResultAssets", +)( + function* ({ + workspaceId, + sourceId, + job, + client, + blobStore = vercelBlobStore, + }: StoreParsedResultAssetsInput) { + const parseResult = (yield* Effect.tryPromise(() => + client.jobs.load(job), + )) as ParsedResultWithAssets + const blobPrefix = getParsedResultBlobPrefix(workspaceId, sourceId) + const resultBlob = yield* Effect.tryPromise(() => + blobStore.put( + `${blobPrefix}/result.zip`, + parseResult.rawZip, + getBlobPutOptions("application/zip"), + ), + ) + + const assetUrlsByFilePath: Record = {} + + for (const image of parseResult.imageChunks ?? []) { + const filePath = normalizeParsedAssetPath(image.filePath) + if (!filePath || !image.data) continue + + const blob = yield* Effect.tryPromise(() => + blobStore.put( + `${blobPrefix}/${filePath}`, + image.data!, + getBlobPutOptions(getContentTypeForPath(filePath)), + ), + ) + assetUrlsByFilePath[filePath] = blob.url + } + + for (const table of parseResult.tableChunks ?? []) { + const filePath = normalizeParsedAssetPath(table.filePath) + if (!filePath || typeof table.html !== "string") continue + + const blob = yield* Effect.tryPromise(() => + blobStore.put( + `${blobPrefix}/${filePath}`, + table.html!, + getBlobPutOptions("text/html; charset=utf-8"), + ), + ) + assetUrlsByFilePath[filePath] = blob.url + } + + return { + resultBlobUrl: resultBlob.url, + assetUrlsByFilePath, + } + }, +) + +// --------------------------------------------------------------------------- +// Async wrapper (backward-compatible) +// --------------------------------------------------------------------------- + export async function storeParsedResultAssets({ workspaceId, sourceId, @@ -59,46 +127,21 @@ export async function storeParsedResultAssets({ client, blobStore = vercelBlobStore, }: StoreParsedResultAssetsInput): Promise { - const parseResult = (await client.jobs.load(job)) as ParsedResultWithAssets - const blobPrefix = getParsedResultBlobPrefix(workspaceId, sourceId) - const resultBlob = await blobStore.put( - `${blobPrefix}/result.zip`, - parseResult.rawZip, - getBlobPutOptions("application/zip"), + return Effect.runPromise( + storeParsedResultAssetsEffect({ + workspaceId, + sourceId, + job, + client, + blobStore, + }), ) - - const assetUrlsByFilePath: Record = {} - - for (const image of parseResult.imageChunks ?? []) { - const filePath = normalizeParsedAssetPath(image.filePath) - if (!filePath || !image.data) continue - - const blob = await blobStore.put( - `${blobPrefix}/${filePath}`, - image.data, - getBlobPutOptions(getContentTypeForPath(filePath)), - ) - assetUrlsByFilePath[filePath] = blob.url - } - - for (const table of parseResult.tableChunks ?? []) { - const filePath = normalizeParsedAssetPath(table.filePath) - if (!filePath || typeof table.html !== "string") continue - - const blob = await blobStore.put( - `${blobPrefix}/${filePath}`, - table.html, - getBlobPutOptions("text/html; charset=utf-8"), - ) - assetUrlsByFilePath[filePath] = blob.url - } - - return { - resultBlobUrl: resultBlob.url, - assetUrlsByFilePath, - } } +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + function getParsedResultBlobPrefix( workspaceId: string, sourceId: string, diff --git a/src/domains/sources/reconcile.ts b/src/domains/sources/reconcile.ts index 6a27fdd..e71ea9a 100644 --- a/src/domains/sources/reconcile.ts +++ b/src/domains/sources/reconcile.ts @@ -1,5 +1,6 @@ import "server-only" +import { Effect, pipe } from "effect" import { del } from "@vercel/blob" import type Knowhere from "@ontos-ai/knowhere-sdk" import type { JobResult } from "@ontos-ai/knowhere-sdk" @@ -29,34 +30,65 @@ type SourceReconcileDependencies = { ) => Promise } +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +export const reconcileSourcesForWorkspaceEffect = Effect.fn( + "reconcileSourcesForWorkspace", +)( + function* ( + workspace: { readonly id: string }, + client: Knowhere, + deps: SourceReconcileDependencies = {}, + ) { + const rows = yield* Effect.tryPromise(() => + sourceWorkflowRuntime.listForWorkspace(workspace.id), + ) + const parsing = rows.filter( + (row) => row.status === "parsing" && row.knowhereJobId, + ) + if (parsing.length === 0) return rows + + yield* pipe( + parsing, + Effect.forEach( + (source) => + Effect.gen(function* () { + const jobId = source.knowhereJobId! + const job = yield* Effect.tryPromise(() => client.jobs.get(jobId)) + yield* Effect.tryPromise(() => + updateSourceFromJob(workspace.id, source, job, client, deps), + ) + }).pipe(Effect.catchAllCause(() => Effect.void)), + { concurrency: "unbounded" }, + ), + ) + + return yield* Effect.tryPromise(() => + sourceWorkflowRuntime.listForWorkspace(workspace.id), + ) + }, +) + +// --------------------------------------------------------------------------- +// Async wrapper (backward-compatible) +// --------------------------------------------------------------------------- + export async function reconcileSourcesForWorkspace( workspace: { readonly id: string }, client: Knowhere, deps: SourceReconcileDependencies = {}, ): Promise { - const rows = await sourceWorkflowRuntime.listForWorkspace(workspace.id) - const parsing = rows.filter( - (row) => row.status === "parsing" && row.knowhereJobId, + return Effect.runPromise( + reconcileSourcesForWorkspaceEffect(workspace, client, deps), ) - if (parsing.length === 0) return rows - - await Promise.all( - parsing.map(async (source) => { - const jobId = source.knowhereJobId - if (!jobId) return - - try { - const job = await client.jobs.get(jobId) - await updateSourceFromJob(workspace.id, source, job, client, deps) - } catch { - // Leave the current row as-is on transient API errors. - } - }), - ) - - return await sourceWorkflowRuntime.listForWorkspace(workspace.id) } +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + async function updateSourceFromJob( workspaceId: string, source: Source, diff --git a/src/domains/sources/route-archive.ts b/src/domains/sources/route-archive.ts index 5a85df5..2b570b6 100644 --- a/src/domains/sources/route-archive.ts +++ b/src/domains/sources/route-archive.ts @@ -1,3 +1,5 @@ +import { Effect } from "effect" + import { routeResult } from "@/lib/route-result" import { getClientForWorkspace } from "./route-dependencies" import type { @@ -26,56 +28,67 @@ type RouteArchive = { function createRouteArchive(deps: RouteArchiveDependencies): RouteArchive { return { - archiveSource: (input: ArchiveSourceInput) => archiveSource(input, deps), + archiveSource: (input: ArchiveSourceInput) => + Effect.runPromise(archiveSourceEffect(input, deps)), } } -async function archiveSource( +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const archiveSourceEffect = ( input: ArchiveSourceInput, deps: RouteArchiveDependencies, -): Promise> { - const user = await deps.requireUser() - const workspace = await deps.ensureWorkspace(user.id) - const source = await deps.sourceService.findInWorkspace( - workspace.id, - input.sourceId, - ) - - if (!source) { - const catalog = await deps.demoApi.fetchCatalog() - const isDemoSource = catalog.sources.some( - (candidate) => candidate.demoSourceId === input.sourceId, +) => + Effect.gen(function* () { + const user = yield* Effect.tryPromise(() => deps.requireUser()) + const workspace = yield* Effect.tryPromise(() => + deps.ensureWorkspace(user.id), + ) + const source = yield* Effect.tryPromise(() => + deps.sourceService.findInWorkspace(workspace.id, input.sourceId), ) - if (isDemoSource) { - await deps.sourceService.hideDemoSource(workspace.id, input.sourceId) - return routeResult.ok({ id: input.sourceId, archived: true }) + + if (!source) { + const catalog = yield* Effect.tryPromise(() => deps.demoApi.fetchCatalog()) + const isDemoSource = catalog.sources.some( + (candidate) => candidate.demoSourceId === input.sourceId, + ) + if (isDemoSource) { + yield* Effect.tryPromise(() => + deps.sourceService.hideDemoSource(workspace.id, input.sourceId), + ) + return routeResult.ok({ id: input.sourceId, archived: true as const }) + } + + return routeResult.error(404, "Source not found.") } - return routeResult.error(404, "Source not found.") - } + if (source.knowhereDocumentId) { + const client = yield* Effect.tryPromise(() => + getClientForWorkspace(workspace.id, input.cookieHeader, deps), + ) + yield* Effect.tryPromise(() => + client.documents.archive(source.knowhereDocumentId!), + ) + } - if (source.knowhereDocumentId) { - const client = await getClientForWorkspace( - workspace.id, - input.cookieHeader, - deps, + yield* Effect.tryPromise(() => + deps.sourceService.softDelete(workspace.id, input.sourceId), ) - await client.documents.archive(source.knowhereDocumentId) - } - - await deps.sourceService.softDelete(workspace.id, input.sourceId) - if (source.demoKey) { - await deps.sourceService.hideDemoSource(workspace.id, source.demoKey) - } - if (source.originalBlobPathname) { - try { - await deps.deleteBlob(source.originalBlobPathname) - } catch { - // Source archival already succeeded; Blob cleanup is best-effort. + if (source.demoKey) { + yield* Effect.tryPromise(() => + deps.sourceService.hideDemoSource(workspace.id, source.demoKey!), + ) + } + if (source.originalBlobPathname) { + yield* Effect.tryPromise(() => + deps.deleteBlob(source.originalBlobPathname!), + ).pipe(Effect.catchAllCause(() => Effect.void)) } - } - return routeResult.ok({ id: input.sourceId, archived: true }) -} + return routeResult.ok({ id: input.sourceId, archived: true as const }) + }) export { createRouteArchive } diff --git a/src/domains/sources/route-chunks.ts b/src/domains/sources/route-chunks.ts index 2e7bad9..6404041 100644 --- a/src/domains/sources/route-chunks.ts +++ b/src/domains/sources/route-chunks.ts @@ -33,72 +33,87 @@ type RouteChunks = { function createRouteChunks(deps: RouteChunksDependencies): RouteChunks { return { loadSourceChunks: (input: LoadSourceChunksInput) => - loadSourceChunks(input, deps), + Effect.runPromise(loadSourceChunksEffect(input, deps)), } } -async function loadSourceChunks( +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const loadSourceChunksEffect = ( input: LoadSourceChunksInput, deps: RouteChunksDependencies, -): Promise> { - const user = await deps.getCurrentUser() - if (!user) { - return (await loadDemoChunkPage(input, deps)) ?? sourceNotFound() - } - - const workspace = await deps.ensureWorkspace(user.id) - const source = await deps.sourceService.findInWorkspace( - workspace.id, - input.sourceId, - ) +) => + Effect.gen(function* () { + const user = yield* Effect.tryPromise(() => deps.getCurrentUser()) + if (!user) { + const demoResult = yield* loadDemoChunkPageEffect(input, deps) + return demoResult ?? sourceNotFound() + } - if (!source) { - return (await loadDemoChunkPage(input, deps)) ?? sourceNotFound() - } + const workspace = yield* Effect.tryPromise(() => + deps.ensureWorkspace(user.id), + ) + const source = yield* Effect.tryPromise(() => + deps.sourceService.findInWorkspace(workspace.id, input.sourceId), + ) - if (source.demoKey) { - return (await loadDemoChunkPage(input, deps, source.demoKey)) ?? sourceNotFound() - } + if (!source) { + const demoResult = yield* loadDemoChunkPageEffect(input, deps) + return demoResult ?? sourceNotFound() + } - const client = await getClientForWorkspace( - workspace.id, - input.cookieHeader, - deps, - ) - const assetUrlsByFilePath = await deps.sourceService.getParseAssetUrls( - workspace.id, - source.id, - ) + if (source.demoKey) { + const demoResult = yield* loadDemoChunkPageEffect( + input, + deps, + source.demoKey, + ) + return demoResult ?? sourceNotFound() + } - if (input.shouldLoadAll) { - const chunks = await Effect.runPromise( - deps.loadChunksForSource(source, client, { assetUrlsByFilePath }), + const client = yield* Effect.tryPromise(() => + getClientForWorkspace(workspace.id, input.cookieHeader, deps), + ) + const assetUrlsByFilePath = yield* Effect.tryPromise(() => + deps.sourceService.getParseAssetUrls(workspace.id, source.id), ) - return routeResult.ok({ chunks }) - } - const chunkPage = await Effect.runPromise( - deps.loadChunkPageForSource(source, client, input.pageParams, { - assetUrlsByFilePath, - }), - ) - return routeResult.ok(chunkPage) -} + if (input.shouldLoadAll) { + const chunks = yield* deps.loadChunksForSource(source, client, { + assetUrlsByFilePath, + }) + return routeResult.ok({ chunks }) + } + + const chunkPage = yield* deps.loadChunkPageForSource( + source, + client, + input.pageParams, + { assetUrlsByFilePath }, + ) + return routeResult.ok(chunkPage) + }) -async function loadDemoChunkPage( +const loadDemoChunkPageEffect = ( input: LoadSourceChunksInput, deps: RouteChunksDependencies, demoSourceId: string = input.sourceId, -): Promise | null> { - try { +) => + Effect.gen(function* () { const pages = input.shouldLoadAll - ? await loadAllDemoChunkPages(input, deps, demoSourceId) + ? yield* Effect.tryPromise(() => + loadAllDemoChunkPages(input, deps, demoSourceId), + ) : [ - await deps.demoApi.fetchChunkPage({ - demoSourceId, - page: input.pageParams.page, - pageSize: input.pageParams.pageSize, - }), + yield* Effect.tryPromise(() => + deps.demoApi.fetchChunkPage({ + demoSourceId, + page: input.pageParams.page, + pageSize: input.pageParams.pageSize, + }), + ), ] const page = pages[0] if (!page) return null @@ -125,19 +140,22 @@ async function loadDemoChunkPage( pagination: page.pagination, }, ) - } catch (error) { - logger.warn("sources: demo chunk load failed", { - sourceId: input.sourceId, - demoSourceId, - page: input.pageParams.page, - pageSize: input.pageParams.pageSize, - shouldLoadAll: input.shouldLoadAll, - knowhereBaseUrl: process.env.KNOWHERE_BASE_URL ?? "(default)", - error: error instanceof Error ? error.message : String(error), - }) - return null - } -} + }).pipe( + Effect.catchAll((error) => + Effect.sync(() => { + logger.warn("sources: demo chunk load failed", { + sourceId: input.sourceId, + demoSourceId, + page: input.pageParams.page, + pageSize: input.pageParams.pageSize, + shouldLoadAll: input.shouldLoadAll, + knowhereBaseUrl: process.env.KNOWHERE_BASE_URL ?? "(default)", + error: error instanceof Error ? error.message : String(error), + }) + return null + }), + ), + ) async function loadAllDemoChunkPages( input: LoadSourceChunksInput, diff --git a/src/domains/sources/route-listing.ts b/src/domains/sources/route-listing.ts index 434dd3a..1daff9c 100644 --- a/src/domains/sources/route-listing.ts +++ b/src/domains/sources/route-listing.ts @@ -26,7 +26,10 @@ type RouteListingDependencies = Pick< | "listSourcesForWorkspace" | "makeKnowhereClient" > & { - readonly demoApi: Pick + readonly demoApi: Pick< + SourceRouteServiceDependencies["demoApi"], + "fetchCatalog" + > readonly sourceService: Pick< SourceRouteServiceDependencies["sourceService"], "listHiddenDemoSourceIds" @@ -41,68 +44,85 @@ type RouteListing = { function createRouteListing(deps: RouteListingDependencies): RouteListing { return { - listSources: (input: ListSourcesInput) => listSources(input, deps), + listSources: (input: ListSourcesInput) => + Effect.runPromise(listSourcesEffect(input, deps)), } } -async function listSources( +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const listSourcesEffect = ( input: ListSourcesInput, deps: RouteListingDependencies, -): Promise> { - const user = await deps.getCurrentUser() - if (!user) { - const catalog = await deps.demoApi.fetchCatalog() - return routeResult.ok({ sources: catalog.sources.map(demoView.toSourceView) }) - } +) => + Effect.gen(function* () { + const user = yield* Effect.tryPromise(() => deps.getCurrentUser()) + if (!user) { + const catalog = yield* Effect.tryPromise(() => deps.demoApi.fetchCatalog()) + return routeResult.ok({ + sources: catalog.sources.map(demoView.toSourceView), + }) + } - const catalog = await knowhereDemoApi.fetchOptionalCatalog(deps.demoApi.fetchCatalog) - const workspace = await deps.ensureWorkspace(user.id) - const sources = await deps.listSourcesForWorkspace(workspace.id) - const demoSourceResolution = resolveWorkspaceDemoSources(sources, catalog) - const sourcesNeedingKnowhereChunkCount = - getWorkspaceSourcesNeedingKnowhereChunkCount( - demoSourceResolution.workspaceSources, + const catalog = yield* Effect.tryPromise(() => + knowhereDemoApi.fetchOptionalCatalog(deps.demoApi.fetchCatalog), + ) + const workspace = yield* Effect.tryPromise(() => + deps.ensureWorkspace(user.id), + ) + const sources = yield* Effect.tryPromise(() => + deps.listSourcesForWorkspace(workspace.id), ) - const materializedDemoSourceOptions = - getMaterializedDemoSourceViewOptionsBySourceId( - demoSourceResolution.workspaceSources, - catalog, + const demoSourceResolution = resolveWorkspaceDemoSources(sources, catalog) + const sourcesNeedingKnowhereChunkCount = + getWorkspaceSourcesNeedingKnowhereChunkCount( + demoSourceResolution.workspaceSources, + ) + const materializedDemoSourceOptions = + getMaterializedDemoSourceViewOptionsBySourceId( + demoSourceResolution.workspaceSources, + catalog, + ) + const apiKey = yield* Effect.tryPromise(() => + deps.ensureApiKeyForWorkspace(workspace.id, input.cookieHeader), ) - const apiKey = await deps.ensureApiKeyForWorkspace( - workspace.id, - input.cookieHeader, - ) - const client = deps.makeKnowhereClient(apiKey) - void reconcileStaleSources(workspace.id, apiKey) - const sourceOptions = await Effect.runPromise( - deps.getSourceViewOptionsBySourceId( + const client = deps.makeKnowhereClient(apiKey) + yield* Effect.fork( + Effect.tryPromise(() => reconcileStaleSources(workspace.id, apiKey)), + ) + const sourceOptions = yield* deps.getSourceViewOptionsBySourceId( sourcesNeedingKnowhereChunkCount, client, - ), - ) - const hiddenDemoSourceIds = new Set( - await deps.sourceService.listHiddenDemoSourceIds(workspace.id), - ) - const visibleDemoSources = catalog.sources - .filter( - (source) => - !demoSourceResolution.materializedDemoSourceIds.has(source.demoSourceId), ) - .filter((source) => !hiddenDemoSourceIds.has(source.demoSourceId)) - .map(demoView.toSourceView) + const hiddenDemoSourceIds = new Set( + yield* Effect.tryPromise(() => + deps.sourceService.listHiddenDemoSourceIds(workspace.id), + ), + ) + const visibleDemoSources = catalog.sources + .filter( + (source) => + !demoSourceResolution.materializedDemoSourceIds.has( + source.demoSourceId, + ), + ) + .filter((source) => !hiddenDemoSourceIds.has(source.demoSourceId)) + .map(demoView.toSourceView) - return routeResult.ok({ - sources: [ - ...visibleDemoSources, - ...demoSourceResolution.workspaceSources.map((source) => - toSourceView( - source, - materializedDemoSourceOptions.get(source.id) ?? - sourceOptions.get(source.id), + return routeResult.ok({ + sources: [ + ...visibleDemoSources, + ...demoSourceResolution.workspaceSources.map((source) => + toSourceView( + source, + materializedDemoSourceOptions.get(source.id) ?? + sourceOptions.get(source.id), + ), ), - ), - ], + ], + }) }) -} export { createRouteListing } diff --git a/src/domains/sources/route-upload.ts b/src/domains/sources/route-upload.ts index f8404e0..2f6f7f4 100644 --- a/src/domains/sources/route-upload.ts +++ b/src/domains/sources/route-upload.ts @@ -1,3 +1,5 @@ +import { Effect } from "effect" + import type { Source, Workspace } from "@/infrastructure/db/schema" import { routeResult } from "@/lib/route-result" import { startBackgroundReconciliation } from "./background-reconcile" @@ -30,60 +32,81 @@ type RouteUpload = { function createRouteUpload(deps: RouteUploadDependencies): RouteUpload { return { - uploadSource: (input: UploadSourceInput) => uploadSource(input, deps), + uploadSource: (input: UploadSourceInput) => + Effect.runPromise(uploadSourceEffect(input, deps)), } } -async function uploadSource( +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const uploadSourceEffect = ( input: UploadSourceInput, deps: RouteUploadDependencies, -): Promise> { - const user = await deps.getCurrentUser() - if (!user) { - return routeResult.error(401, "Please log in to upload documents.") - } +) => + Effect.gen(function* () { + const user = yield* Effect.tryPromise(() => deps.getCurrentUser()) + if (!user) { + return routeResult.error(401, "Please log in to upload documents.") + } - if (input.upload.type === "error") { - return routeResult.badRequest(input.upload.message) - } + if (input.upload.type === "error") { + return routeResult.badRequest(input.upload.message) + } - const validation = - input.upload.type === "file" - ? validateUploadFile(input.upload.file) - : validateSourceBlobUploadInput(input.upload.input) - if (!validation.ok) { - return routeResult.badRequest(validation.message) - } + const validation = + input.upload.type === "file" + ? validateUploadFile(input.upload.file) + : validateSourceBlobUploadInput(input.upload.input) + if (!validation.ok) { + return routeResult.badRequest(validation.message) + } - const workspace = await deps.ensureWorkspace(user.id) - const apiKey = await deps.ensureApiKeyForWorkspace( - workspace.id, - input.cookieHeader, - ) - const client = deps.makeKnowhereClient(apiKey) - const source = await uploadToKnowhere(workspace, input.upload, client, deps) - .finally(() => { - input.onUploadFinished?.() - }) + const workspace = yield* Effect.tryPromise(() => + deps.ensureWorkspace(user.id), + ) + const apiKey = yield* Effect.tryPromise(() => + deps.ensureApiKeyForWorkspace(workspace.id, input.cookieHeader), + ) + const client = deps.makeKnowhereClient(apiKey) - startBackgroundReconciliation(workspace.id, source.id, apiKey) + const source = yield* uploadToKnowhereEffect( + workspace, + input.upload, + client, + deps, + ).pipe( + Effect.onExit(() => + Effect.sync(() => { + input.onUploadFinished?.() + }), + ), + ) - return routeResult.ok({ source: toSourceView(source) }, 201) -} + yield* Effect.tryPromise(() => + startBackgroundReconciliation(workspace.id, source.id, apiKey), + ) + + return routeResult.ok({ source: toSourceView(source) }, 201) + }) -async function uploadToKnowhere( +const uploadToKnowhereEffect = ( workspace: Workspace, upload: Exclude, client: SourceRouteKnowhereClient, deps: RouteUploadDependencies, -): Promise { - return upload.type === "file" - ? deps.sourceService.uploadSourceToKnowhere(workspace, upload.file, client) - : deps.sourceService.uploadSourceBlobToKnowhere( - workspace, - upload.input, - client, +) => + upload.type === "file" + ? Effect.tryPromise(() => + deps.sourceService.uploadSourceToKnowhere(workspace, upload.file, client), + ) + : Effect.tryPromise(() => + deps.sourceService.uploadSourceBlobToKnowhere( + workspace, + upload.input, + client, + ), ) -} export { createRouteUpload } diff --git a/src/domains/workspace/initial-state.ts b/src/domains/workspace/initial-state.ts index c5f0843..b7e05fa 100644 --- a/src/domains/workspace/initial-state.ts +++ b/src/domains/workspace/initial-state.ts @@ -53,7 +53,10 @@ type WorkspaceShellInitialStateDependencies = { readonly fetchDemoCatalog: () => Promise readonly getClientForWorkspace: ( workspace: Workspace, - ) => Promise<{ readonly apiKey: string; readonly client: WorkspaceShellInitialStateClient }> + ) => Promise<{ + readonly apiKey: string + readonly client: WorkspaceShellInitialStateClient + }> readonly getGuest: () => Promise<{ readonly loginUrl: string }> readonly getOptionalAuthenticated: () => Promise<{ readonly user: AuthUser @@ -66,13 +69,17 @@ type WorkspaceShellInitialStateDependencies = { readonly thread: ChatThread readonly messages: readonly ChatMessage[] } | null> - readonly listChatThreads: (workspaceId: string) => Promise + readonly listChatThreads: ( + workspaceId: string, + ) => Promise readonly listHiddenDemoSourceIds: (workspaceId: string) => Promise readonly listMessages: ( workspaceId: string, threadId: string, ) => Promise - readonly listSourcesForWorkspace: (workspaceId: string) => Promise + readonly listSourcesForWorkspace: ( + workspaceId: string, + ) => Promise readonly sourceViewOptionsBySourceId: ( sources: readonly Source[], client: WorkspaceShellInitialStateClient, @@ -92,101 +99,141 @@ const defaultDependencies: WorkspaceShellInitialStateDependencies = { sourceViewOptionsBySourceId: getSourceViewOptionsBySourceId, } -export async function loadWorkspaceShellInitialState( +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +export const loadWorkspaceShellInitialStateEffect = ( deps: WorkspaceShellInitialStateDependencies = defaultDependencies, -): Promise { - const context = await deps.getOptionalAuthenticated() +) => + Effect.gen(function* () { + const context = yield* Effect.tryPromise(() => + deps.getOptionalAuthenticated(), + ) - if (!context) { - const demoCatalog = await deps.fetchDemoCatalog() - const guestContext = await deps.getGuest() - return { - isGuest: true, - sources: demoCatalog.sources.map(demoView.toSourceView), - chatMessages: demoView.toChatMessages(demoCatalog), - dashboardUrl: resolveDashboardUrl(), - loginUrl: guestContext.loginUrl, + if (!context) { + const demoCatalog = yield* Effect.tryPromise(() => + deps.fetchDemoCatalog(), + ) + const guestContext = yield* Effect.tryPromise(() => deps.getGuest()) + return { + isGuest: true, + sources: demoCatalog.sources.map(demoView.toSourceView), + chatMessages: demoView.toChatMessages(demoCatalog), + dashboardUrl: resolveDashboardUrl(), + loginUrl: guestContext.loginUrl, + } } - } - const { user, workspace } = context - const demoCatalog = await knowhereDemoApi.fetchOptionalCatalog(deps.fetchDemoCatalog) - const sources = await deps.listSourcesForWorkspace(workspace.id) - const demoSourceResolution = resolveWorkspaceDemoSources( - sources, - demoCatalog, - ) - const hiddenDemoSourceIds = new Set( - await deps.listHiddenDemoSourceIds(workspace.id), - ) - const visibleDemoCatalogSources = demoCatalog.sources - .filter( - (source) => - !demoSourceResolution.materializedDemoSourceIds.has(source.demoSourceId), + const { user, workspace } = context + const demoCatalog = yield* Effect.tryPromise(() => + knowhereDemoApi.fetchOptionalCatalog(deps.fetchDemoCatalog), + ) + const sources = yield* Effect.tryPromise(() => + deps.listSourcesForWorkspace(workspace.id), + ) + const demoSourceResolution = resolveWorkspaceDemoSources( + sources, + demoCatalog, + ) + const hiddenDemoSourceIds = new Set( + yield* Effect.tryPromise(() => + deps.listHiddenDemoSourceIds(workspace.id), + ), ) - .filter((source) => !hiddenDemoSourceIds.has(source.demoSourceId)) - const demoSources = visibleDemoCatalogSources.map(demoView.toSourceView) - const listedChatThreads = await deps.listChatThreads(workspace.id) - const seededDemoChatThread = - listedChatThreads.length === 0 - ? await deps.ensureDemoChatThread(workspace.id, demoCatalog) - : null - const chatThreads = seededDemoChatThread - ? [seededDemoChatThread.thread] - : listedChatThreads - const activeChatThread = chatThreads[0] ?? null - const activeChatMessages = seededDemoChatThread - ? seededDemoChatThread.messages - : activeChatThread - ? await deps.listMessages(workspace.id, activeChatThread.id) + const visibleDemoCatalogSources = demoCatalog.sources + .filter( + (source) => + !demoSourceResolution.materializedDemoSourceIds.has( + source.demoSourceId, + ), + ) + .filter((source) => !hiddenDemoSourceIds.has(source.demoSourceId)) + const demoSources = visibleDemoCatalogSources.map(demoView.toSourceView) + const listedChatThreads = yield* Effect.tryPromise(() => + deps.listChatThreads(workspace.id), + ) + const seededDemoChatThread = + listedChatThreads.length === 0 + ? yield* Effect.tryPromise(() => + deps.ensureDemoChatThread(workspace.id, demoCatalog), + ) + : null + const chatThreads = seededDemoChatThread + ? [seededDemoChatThread.thread] + : listedChatThreads + const activeChatThread = chatThreads[0] ?? null + const activeChatMessages = seededDemoChatThread + ? seededDemoChatThread.messages + : activeChatThread + ? yield* Effect.tryPromise(() => + deps.listMessages(workspace.id, activeChatThread.id), + ) + : [] + const chatMessages = activeChatMessages + ? activeChatMessages.map((message) => toChatMessageView(message)) : [] - const chatMessages = activeChatMessages - ? activeChatMessages.map( - (message) => toChatMessageView(message), + const sourcesNeedingKnowhereChunkCount = + getWorkspaceSourcesNeedingKnowhereChunkCount( + demoSourceResolution.workspaceSources, ) - : [] - const sourcesNeedingKnowhereChunkCount = - getWorkspaceSourcesNeedingKnowhereChunkCount( - demoSourceResolution.workspaceSources, + const materializedDemoSourceOptions = + getMaterializedDemoSourceViewOptionsBySourceId( + demoSourceResolution.workspaceSources, + demoCatalog, + ) + const { client, apiKey } = yield* Effect.tryPromise(() => + deps.getClientForWorkspace(workspace), ) - const materializedDemoSourceOptions = - getMaterializedDemoSourceViewOptionsBySourceId( - demoSourceResolution.workspaceSources, - demoCatalog, + yield* Effect.fork( + Effect.tryPromise(() => reconcileStaleSources(workspace.id, apiKey)), + ) + const sourceOptions = yield* deps.sourceViewOptionsBySourceId( + sourcesNeedingKnowhereChunkCount, + client, ) - const { client, apiKey } = await deps.getClientForWorkspace(workspace) - void reconcileStaleSources(workspace.id, apiKey) - const sourceOptions = await Effect.runPromise( - deps.sourceViewOptionsBySourceId(sourcesNeedingKnowhereChunkCount, client), - ) - return { - user: { - id: user.id, - name: user.name ?? null, - email: user.email ?? null, - }, - workspace: { - id: workspace.id, - namespace: workspace.namespace, - }, - dashboardUrl: resolveDashboardUrl(), - sources: [ - ...demoSources, - ...demoSourceResolution.workspaceSources.map((source) => - toSourceView( - source, - materializedDemoSourceOptions.get(source.id) ?? - sourceOptions.get(source.id), + return { + user: { + id: user.id, + name: user.name ?? null, + email: user.email ?? null, + }, + workspace: { + id: workspace.id, + namespace: workspace.namespace, + }, + dashboardUrl: resolveDashboardUrl(), + sources: [ + ...demoSources, + ...demoSourceResolution.workspaceSources.map((source) => + toSourceView( + source, + materializedDemoSourceOptions.get(source.id) ?? + sourceOptions.get(source.id), + ), ), - ), - ], - chatThreads: chatThreads.map(toChatThreadView), - activeChatThreadId: activeChatThread?.id ?? null, - chatMessages, - } + ], + chatThreads: chatThreads.map(toChatThreadView), + activeChatThreadId: activeChatThread?.id ?? null, + chatMessages, + } + }) + +// --------------------------------------------------------------------------- +// Async wrapper (backward-compatible) +// --------------------------------------------------------------------------- + +export async function loadWorkspaceShellInitialState( + deps: WorkspaceShellInitialStateDependencies = defaultDependencies, +): Promise { + return Effect.runPromise(loadWorkspaceShellInitialStateEffect(deps)) } +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + function resolveDashboardUrl(): string | undefined { return process.env.DASHBOARD_ORIGIN } diff --git a/src/domains/workspace/request-context.ts b/src/domains/workspace/request-context.ts index 726502b..dc0e669 100644 --- a/src/domains/workspace/request-context.ts +++ b/src/domains/workspace/request-context.ts @@ -1,10 +1,15 @@ import "server-only" +import { Effect } from "effect" import { headers } from "next/headers" import { ensureApiKeyForWorkspace } from "@/integrations/dashboard/api-key-service" import { authURLs } from "@/infrastructure/auth/urls" -import { getCurrentUser, requireUser, type AuthUser } from "@/infrastructure/auth" +import { + getCurrentUser, + requireUser, + type AuthUser, +} from "@/infrastructure/auth" import { makeKnowhereClient } from "@/integrations/knowhere" import { workspaceService } from "@/domains/workspace/service" import type { Workspace } from "@/infrastructure/db/schema" @@ -25,54 +30,92 @@ type GuestNotebookContext = { readonly loginUrl: string } -async function getAuthenticated(): Promise { - const user = await requireUser() - const workspace = await workspaceService.ensureWorkspace(user.id) +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const getAuthenticatedEffect = Effect.gen(function* () { + const user = yield* Effect.tryPromise(() => requireUser()) + const workspace = yield* Effect.tryPromise(() => + workspaceService.ensureWorkspace(user.id), + ) + + return { user, workspace } + }) + +const getOptionalAuthenticatedEffect = Effect.gen(function* () { + const user = yield* Effect.tryPromise(() => getCurrentUser()) + if (!user) return null + + const workspace = yield* Effect.tryPromise(() => + workspaceService.ensureWorkspace(user.id), + ) + return { user, workspace } + }) + +const getAuthenticatedWithClientEffect = Effect.gen(function* () { + const context = yield* getAuthenticatedEffect + const clientContext = yield* getClientForWorkspaceEffect(context.workspace) + + return { + ...context, + ...clientContext, + } + }) + +const getClientForWorkspaceEffect = (workspace: Workspace) => + Effect.gen(function* () { + const cookieHeader = + (yield* Effect.tryPromise(() => headers())).get("cookie") ?? "" + const apiKey = yield* Effect.tryPromise(() => + ensureApiKeyForWorkspace(workspace.id, cookieHeader), + ) + const client = makeKnowhereClient(apiKey) + + return { apiKey, client } + }) + +const getGuestEffect = Effect.gen(function* () { + const dashboardOrigin = + process.env.DASHBOARD_ORIGIN ?? "http://localhost:3000" + const dashboardLoginURL = `${dashboardOrigin}/login` + const headersList = yield* Effect.tryPromise(() => headers()) + const notebookPublicURL = + process.env.NOTEBOOK_PUBLIC_URL ?? + authURLs.resolveNotebookPublicURLFromHeaders(headersList) + const loginUrl = authURLs.buildDashboardLoginURL( + dashboardLoginURL, + notebookPublicURL, + ) + + return { loginUrl } + }, +) + +// --------------------------------------------------------------------------- +// Async wrappers (backward-compatible) +// --------------------------------------------------------------------------- - return { user, workspace } +async function getAuthenticated(): Promise { + return Effect.runPromise(getAuthenticatedEffect) } async function getOptionalAuthenticated(): Promise { - const user = await getCurrentUser() - if (!user) return null - - const workspace = await workspaceService.ensureWorkspace(user.id) - return { user, workspace } + return Effect.runPromise(getOptionalAuthenticatedEffect) } async function getAuthenticatedWithClient(): Promise { - const context = await getAuthenticated() - const clientContext = await getClientForWorkspace(context.workspace) - - return { - ...context, - ...clientContext, - } + return Effect.runPromise(getAuthenticatedWithClientEffect) } async function getClientForWorkspace( workspace: Workspace, ): Promise> { - const cookieHeader = (await headers()).get("cookie") ?? "" - const apiKey = await ensureApiKeyForWorkspace(workspace.id, cookieHeader) - const client = makeKnowhereClient(apiKey) - - return { apiKey, client } + return Effect.runPromise(getClientForWorkspaceEffect(workspace)) } async function getGuest(): Promise { - const dashboardOrigin = - process.env.DASHBOARD_ORIGIN ?? "http://localhost:3000" - const dashboardLoginURL = `${dashboardOrigin}/login` - const notebookPublicURL = - process.env.NOTEBOOK_PUBLIC_URL ?? - authURLs.resolveNotebookPublicURLFromHeaders(await headers()) - const loginUrl = authURLs.buildDashboardLoginURL( - dashboardLoginURL, - notebookPublicURL, - ) - - return { loginUrl } + return Effect.runPromise(getGuestEffect) } export const notebookRequestContext = { diff --git a/src/integrations/knowhere-demo.ts b/src/integrations/knowhere-demo.ts index 87f2dc9..b10434a 100644 --- a/src/integrations/knowhere-demo.ts +++ b/src/integrations/knowhere-demo.ts @@ -1,5 +1,7 @@ import "server-only" +import { Effect } from "effect" + export type DemoCitation = { readonly demoSourceId: string readonly canonicalDocumentId: string @@ -177,35 +179,109 @@ const DEFAULT_KNOWHERE_BASE_URL = "https://api.knowhereto.ai" const emptyCatalog: DemoCatalog = { sources: [] } -async function fetchOptionalCatalog( - fetcher?: () => Promise, -): Promise { - try { - return await (fetcher ?? fetchCatalog)() - } catch { - return emptyCatalog - } -} +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- -export const knowhereDemoApi = { - fetchCatalog, - fetchOptionalCatalog, - fetchChunkPage, - materializeSources, - resolveApiURL, -} as const - -async function fetchCatalog(): Promise { - const response = await fetch(resolveApiURL("/api/v1/demo/catalog"), { - cache: "force-cache", - next: { revalidate: 300 }, - }) - await assertOk(response) +const fetchCatalogEffect = Effect.fn("knowhereDemo.fetchCatalog")(function* () { + const response = yield* Effect.tryPromise(() => + fetch(resolveApiURL("/api/v1/demo/catalog"), { + cache: "force-cache", + next: { revalidate: 300 }, + }), + ) + yield* assertOkEffect(response) - const body = (await response.json()) as DemoCatalogResponse + const body = (yield* Effect.tryPromise(() => + response.json(), + )) as DemoCatalogResponse return { sources: (body.sources ?? []).map(toDemoSource), } +}) + +const fetchChunkPageEffect = Effect.fn("knowhereDemo.fetchChunkPage")( + function* (input: { + readonly demoSourceId: string + readonly page: number + readonly pageSize: number + }) { + const params = new URLSearchParams({ + page: String(input.page), + page_size: String(input.pageSize), + }) + const response = yield* Effect.tryPromise(() => + fetch( + resolveApiURL( + `/api/v1/demo/sources/${encodeURIComponent(input.demoSourceId)}/chunks?${params.toString()}`, + ), + { cache: "no-store" }, + ), + ) + yield* assertOkEffect(response) + + return toDemoChunkPage( + (yield* Effect.tryPromise(() => + response.json(), + )) as DemoChunkPageResponse, + ) + }, +) + +const materializeSourcesEffect = Effect.fn("knowhereDemo.materializeSources")( + function* (input: { + readonly apiKey: string + readonly namespace: string + readonly demoSourceIds: readonly string[] + }) { + const requestBody = JSON.stringify({ + namespace: input.namespace, + demo_source_ids: input.demoSourceIds, + }) + const response = yield* Effect.tryPromise(() => + fetch(resolveApiURL("/api/v1/demo/materializations"), { + method: "POST", + headers: { + authorization: `Bearer ${input.apiKey}`, + "content-type": "application/json", + }, + body: requestBody, + }), + ) + yield* assertOkEffect(response) + + const body = (yield* Effect.tryPromise(() => + response.json(), + )) as MaterializeResponse + return (body.sources ?? []).map(toMaterializedDemoSource) + }, +) + +const fetchOptionalCatalogEffect = ( + fetcher?: () => Effect.Effect, +) => + (fetcher ?? fetchCatalogEffect)().pipe( + Effect.catchAll(() => Effect.succeed(emptyCatalog)), + ) + +// --------------------------------------------------------------------------- +// Async wrappers (backward-compatible) +// --------------------------------------------------------------------------- + +async function fetchCatalog(): Promise { + return Effect.runPromise(fetchCatalogEffect()) +} + +async function fetchOptionalCatalog( + fetcher?: () => Promise, +): Promise { + const effectFetcher = fetcher + ? () => + Effect.tryPromise(() => fetcher()).pipe( + Effect.catchAll(() => Effect.succeed(emptyCatalog)), + ) + : undefined + return Effect.runPromise(fetchOptionalCatalogEffect(effectFetcher)) } async function fetchChunkPage(input: { @@ -213,19 +289,7 @@ async function fetchChunkPage(input: { readonly page: number readonly pageSize: number }): Promise { - const params = new URLSearchParams({ - page: String(input.page), - page_size: String(input.pageSize), - }) - const response = await fetch( - resolveApiURL( - `/api/v1/demo/sources/${encodeURIComponent(input.demoSourceId)}/chunks?${params.toString()}`, - ), - { cache: "no-store" }, - ) - await assertOk(response) - - return toDemoChunkPage((await response.json()) as DemoChunkPageResponse) + return Effect.runPromise(fetchChunkPageEffect(input)) } async function materializeSources(input: { @@ -233,35 +297,51 @@ async function materializeSources(input: { readonly namespace: string readonly demoSourceIds: readonly string[] }): Promise { - const response = await fetch(resolveApiURL("/api/v1/demo/materializations"), { - method: "POST", - headers: { - authorization: `Bearer ${input.apiKey}`, - "content-type": "application/json", - }, - body: JSON.stringify({ - namespace: input.namespace, - demo_source_ids: input.demoSourceIds, - }), - }) - await assertOk(response) - - const body = (await response.json()) as MaterializeResponse - return (body.sources ?? []).map(toMaterializedDemoSource) + return Effect.runPromise(materializeSourcesEffect(input)) } +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +export const knowhereDemoApi = { + fetchCatalog, + fetchOptionalCatalog, + fetchChunkPage, + materializeSources, + resolveApiURL, +} as const + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + function resolveApiURL(path: string): string { const baseURL = process.env.KNOWHERE_BASE_URL ?? DEFAULT_KNOWHERE_BASE_URL return new URL(path, baseURL).toString() } -async function assertOk(response: Response): Promise { - if (response.ok) return +class KnowhereDemoApiError { + readonly _tag = "KnowhereDemoApiError" + constructor( + readonly status: number, + readonly body: string, + ) {} +} - const body = await response.text().catch(() => "") - throw new Error( - `Knowhere demo API failed: status=${response.status}, body=${body.slice(0, 300)}`, - ) +function assertOkEffect( + response: Response, +): Effect.Effect { + if (response.ok) return Effect.void + + return Effect.gen(function* () { + const body = yield* Effect.tryPromise(() => + response.text().catch(() => ""), + ).pipe(Effect.orDie) + return yield* Effect.fail( + new KnowhereDemoApiError(response.status, body), + ) + }) } function toDemoSource(source: DemoSourceResponse): DemoSource { diff --git a/src/lib/api-error-response.ts b/src/lib/api-error-response.ts index 251ddb1..c16a26e 100644 --- a/src/lib/api-error-response.ts +++ b/src/lib/api-error-response.ts @@ -1,5 +1,6 @@ import "server-only" +import { Effect } from "effect" import { NextResponse } from "next/server" import { formatUnknownForLog } from "./format-log-value" @@ -10,13 +11,20 @@ export async function withApiErrorResponse( handler: () => Promise, fallbackMessage: string = "Something went wrong. Please try again.", ): Promise { - try { - return await handler() - } catch (error) { - logger.error("api: unhandled request failure", { - context, - error: formatUnknownForLog(error), - }) - return NextResponse.json({ message: fallbackMessage }, { status: 500 }) - } + return Effect.runPromise( + Effect.tryPromise(handler).pipe( + Effect.catchAll((error) => + Effect.sync(() => { + logger.error("api: unhandled request failure", { + context, + error: formatUnknownForLog(error), + }) + return NextResponse.json( + { message: fallbackMessage }, + { status: 500 }, + ) + }), + ), + ), + ) } diff --git a/src/lib/route-result.ts b/src/lib/route-result.ts index 5b1bb4a..3ec3167 100644 --- a/src/lib/route-result.ts +++ b/src/lib/route-result.ts @@ -1,3 +1,5 @@ +import { Effect } from "effect" + export type RouteResult = { readonly status: number readonly body: TBody @@ -31,22 +33,39 @@ function badRequest(message: string): RouteResult { return error(400, message) } +// --------------------------------------------------------------------------- +// Effect core +// --------------------------------------------------------------------------- + +const readJsonEffect = ( + request: Request, +): Effect.Effect => + Effect.tryPromise(() => request.json()).pipe( + Effect.map( + (value): ReadJsonResult => ({ ok: true, value }), + ), + Effect.catchAllCause( + (): Effect.Effect => Effect.succeed({ ok: false }), + ), + ) + +const readJsonOrNullEffect = ( + request: Request, +): Effect.Effect => + readJsonEffect(request).pipe( + Effect.map((body) => (body.ok ? body.value : null)), + ) + +// --------------------------------------------------------------------------- +// Async wrappers (backward-compatible) +// --------------------------------------------------------------------------- + async function readJson(request: Request): Promise { - try { - return { - ok: true, - value: await request.json(), - } - } catch { - return { ok: false } - } + return Effect.runPromise(readJsonEffect(request)) } async function readJsonOrNull(request: Request): Promise { - const body = await readJson(request) - if (!body.ok) return null - - return body.value + return Effect.runPromise(readJsonOrNullEffect(request)) } export const routeResult = { From 5f7a1b047b68ad6dfa97ace264e7f8fe3f99cb7b Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 16:12:43 +0800 Subject: [PATCH 11/16] fix: make document reconciliation client-driven with atomic dedup Replace the automatic reconcileStaleSources background sweep with inline, per-source triggers that fire only when the client polls for source state. Add three-layer protection against duplicate workflow runs: Upstash idempotency via workflowRunId, in-memory dedup, and an atomic DB guard (UPDATE WHERE status='parsing') so concurrent workflows cannot both finalize the same source. --- src/domains/sources/background-reconcile.ts | 38 +++++--------------- src/domains/sources/lifecycle.ts | 7 ++++ src/domains/sources/route-listing.ts | 14 +++++--- src/domains/sources/source-row-repository.ts | 31 ++++++++++------ src/domains/sources/workflow-runtime.ts | 4 ++- src/domains/workspace/initial-state.ts | 15 +++++--- 6 files changed, 61 insertions(+), 48 deletions(-) diff --git a/src/domains/sources/background-reconcile.ts b/src/domains/sources/background-reconcile.ts index 8eb98c2..3b2a8c6 100644 --- a/src/domains/sources/background-reconcile.ts +++ b/src/domains/sources/background-reconcile.ts @@ -3,9 +3,16 @@ import "server-only" import { Effect } from "effect" import { Client } from "@upstash/workflow" -import { sourceWorkflowRuntime } from "./workflow-runtime" import { logger } from "@/lib/logger" +// Re-trigger protection: Layers 1 & 2. +// +// Layer 1 — Upstash idempotency via workflowRunId=sourceId ensures at most one +// running workflow per source, even across process restarts or multiple instances. +// +// Layer 2 — In-memory Set avoids the network call entirely when the same process +// already triggered a workflow for this source. + const triggeredSourceIds = new Set() function createClient(): Client { @@ -33,6 +40,7 @@ const startBackgroundReconciliationEffect = ( createClient().trigger({ url: `${resolveBaseURL()}/api/sources/reconcile`, body: { workspaceId, sourceId, apiKey }, + workflowRunId: sourceId, retries: 3, }), ) @@ -51,27 +59,6 @@ const startBackgroundReconciliationEffect = ( ), ) -const reconcileStaleSourcesEffect = ( - workspaceId: string, - apiKey: string, -): Effect.Effect => - Effect.gen(function* () { - const sources = yield* Effect.tryPromise(() => - sourceWorkflowRuntime.listForWorkspace(workspaceId), - ) - for (const source of sources) { - if (source.status === "parsing" && source.knowhereJobId) { - yield* Effect.fork( - startBackgroundReconciliationEffect(workspaceId, source.id, apiKey), - ) - } - } - }).pipe(Effect.catchAllCause(() => Effect.void)) - -// --------------------------------------------------------------------------- -// Async wrappers (backward-compatible) -// --------------------------------------------------------------------------- - export async function startBackgroundReconciliation( workspaceId: string, sourceId: string, @@ -81,10 +68,3 @@ export async function startBackgroundReconciliation( startBackgroundReconciliationEffect(workspaceId, sourceId, apiKey), ) } - -export async function reconcileStaleSources( - workspaceId: string, - apiKey: string, -): Promise { - return Effect.runPromise(reconcileStaleSourcesEffect(workspaceId, apiKey)) -} diff --git a/src/domains/sources/lifecycle.ts b/src/domains/sources/lifecycle.ts index 1bb4f24..780dad2 100644 --- a/src/domains/sources/lifecycle.ts +++ b/src/domains/sources/lifecycle.ts @@ -24,6 +24,7 @@ type SourceLifecycleRepository = { workspaceId: string, sourceId: string, reason: string, + requiredStatus?: string, ): Promise clearSourceStagedBlob(workspaceId: string, sourceId: string): Promise } @@ -64,6 +65,10 @@ export const applyKnowhereJobToSourceEffect = Effect.fn( parsedResultStore, blobStore, }: ApplyKnowhereJobToSourceInput) { + // Best-effort early exit: skip expensive asset uploads when the source has + // already been resolved. The atomic guard (Layer 3) is in the DB UPDATE below. + if (source.status !== "parsing") return + if (job.isDone || job.status === "done") { if (job.documentId) { const stored = yield* Effect.tryPromise(() => @@ -94,6 +99,7 @@ export const applyKnowhereJobToSourceEffect = Effect.fn( workspaceId, source.id, "Parsing finished but no document was published.", + "parsing", ), ) yield* cleanupStagedBlobEffect(workspaceId, source, repository, blobStore) @@ -106,6 +112,7 @@ export const applyKnowhereJobToSourceEffect = Effect.fn( workspaceId, source.id, job.error?.message ?? "Parsing failed.", + "parsing", ), ) yield* cleanupStagedBlobEffect(workspaceId, source, repository, blobStore) diff --git a/src/domains/sources/route-listing.ts b/src/domains/sources/route-listing.ts index 1daff9c..b204969 100644 --- a/src/domains/sources/route-listing.ts +++ b/src/domains/sources/route-listing.ts @@ -9,7 +9,7 @@ import { import { routeResult } from "@/lib/route-result" import { knowhereDemoApi } from "@/integrations/knowhere-demo" import { toSourceView } from "./view" -import { reconcileStaleSources } from "./background-reconcile" +import { startBackgroundReconciliation } from "./background-reconcile" import type { JsonRouteResult, ListSourcesBody, @@ -89,9 +89,15 @@ const listSourcesEffect = ( deps.ensureApiKeyForWorkspace(workspace.id, input.cookieHeader), ) const client = deps.makeKnowhereClient(apiKey) - yield* Effect.fork( - Effect.tryPromise(() => reconcileStaleSources(workspace.id, apiKey)), - ) + for (const source of sources) { + if (source.status === "parsing" && source.knowhereJobId) { + yield* Effect.fork( + Effect.tryPromise(() => + startBackgroundReconciliation(workspace.id, source.id, apiKey), + ), + ) + } + } const sourceOptions = yield* deps.getSourceViewOptionsBySourceId( sourcesNeedingKnowhereChunkCount, client, diff --git a/src/domains/sources/source-row-repository.ts b/src/domains/sources/source-row-repository.ts index 5f25249..71b054a 100644 --- a/src/domains/sources/source-row-repository.ts +++ b/src/domains/sources/source-row-repository.ts @@ -59,6 +59,7 @@ type SourceRowRepository = { workspaceId: string, sourceId: string, reason: string, + requiredStatus?: string, ) => Effect.Effect readonly clearStagedBlobEffect: ( workspaceId: string, @@ -79,6 +80,7 @@ type SourceRowRepository = { workspaceId: string, sourceId: string, values: SourceUpdate, + requiredStatus?: string, ) => Promise readonly requireSource: (source: Source | null, message: string) => Source } @@ -162,17 +164,18 @@ const markReadyEffect: SourceRowRepository["markReadyEffect"] = ( status: "ready", knowhereDocumentId: documentId, failureReason: null, - }) + }, "parsing") const markFailedEffect: SourceRowRepository["markFailedEffect"] = ( workspaceId: string, sourceId: string, reason: string, + requiredStatus?: string, ) => updateInWorkspaceEffect(workspaceId, sourceId, { status: "failed", failureReason: reason, - }) + }, requiredStatus) const clearStagedBlobEffect: SourceRowRepository["clearStagedBlobEffect"] = ( workspaceId: string, @@ -212,11 +215,12 @@ const updateInWorkspaceEffect = ( workspaceId: string, sourceId: string, values: SourceUpdate, + requiredStatus?: string, ) => Effect.gen(function* () { const db = yield* DbClient return yield* Effect.promise(() => - updateInWorkspaceWithDb(db, workspaceId, sourceId, values), + updateInWorkspaceWithDb(db, workspaceId, sourceId, values, requiredStatus), ) }) @@ -247,19 +251,26 @@ async function updateInWorkspaceWithDb( workspaceId: string, sourceId: string, values: SourceUpdate, + requiredStatus?: string, ): Promise { if (!isWorkspaceSourceId(sourceId)) return null + // Layer 3 — Atomic status guard. + // When requiredStatus is set, the UPDATE only matches if the source is still in + // the expected status. Two concurrent workflows will race; only one wins. + const conditions = [ + eq(sources.id, sourceId), + eq(sources.workspaceId, workspaceId), + isNull(sources.deletedAt), + ] + if (requiredStatus) { + conditions.push(eq(sources.status, requiredStatus)) + } + const [source] = await db .update(sources) .set({ ...values, updatedAt: sql`now()` }) - .where( - and( - eq(sources.id, sourceId), - eq(sources.workspaceId, workspaceId), - isNull(sources.deletedAt), - ), - ) + .where(and(...conditions)) .returning() return source ?? null diff --git a/src/domains/sources/workflow-runtime.ts b/src/domains/sources/workflow-runtime.ts index b713f58..fff2202 100644 --- a/src/domains/sources/workflow-runtime.ts +++ b/src/domains/sources/workflow-runtime.ts @@ -31,6 +31,7 @@ type UploadRepositoryRuntime = { workspaceId: string, sourceId: string, reason: string, + requiredStatus?: string, ) => Promise } @@ -133,9 +134,10 @@ const markFailed: SourceWorkflowRuntime["markFailed"] = ( workspaceId: string, sourceId: string, reason: string, + requiredStatus?: string, ) => databaseRuntime.runPromise( - sourceRepository.markFailedEffect(workspaceId, sourceId, reason), + sourceRepository.markFailedEffect(workspaceId, sourceId, reason, requiredStatus), ) const clearStagedBlob: SourceWorkflowRuntime["clearStagedBlob"] = ( diff --git a/src/domains/workspace/initial-state.ts b/src/domains/workspace/initial-state.ts index b7e05fa..39f2fc5 100644 --- a/src/domains/workspace/initial-state.ts +++ b/src/domains/workspace/initial-state.ts @@ -13,8 +13,9 @@ import { chatThreadService } from "@/domains/chat/thread-service" import { toChatMessageView, toChatThreadView } from "@/domains/chat/view" import { sourceViewOptionsBySourceId as getSourceViewOptionsBySourceId } from "@/domains/sources/counts" import { sourceService } from "@/domains/sources/service" +import { startBackgroundReconciliation } from "@/domains/sources/background-reconcile" import { sourceWorkflowRuntime } from "@/domains/sources/workflow-runtime" -import { reconcileStaleSources } from "@/domains/sources/background-reconcile" + import type { SourceView } from "@/domains/sources/types" import { toSourceView } from "@/domains/sources/view" import type { AuthUser } from "@/infrastructure/auth" @@ -185,9 +186,15 @@ export const loadWorkspaceShellInitialStateEffect = ( const { client, apiKey } = yield* Effect.tryPromise(() => deps.getClientForWorkspace(workspace), ) - yield* Effect.fork( - Effect.tryPromise(() => reconcileStaleSources(workspace.id, apiKey)), - ) + for (const source of sources) { + if (source.status === "parsing" && source.knowhereJobId) { + yield* Effect.fork( + Effect.tryPromise(() => + startBackgroundReconciliation(workspace.id, source.id, apiKey), + ), + ) + } + } const sourceOptions = yield* deps.sourceViewOptionsBySourceId( sourcesNeedingKnowhereChunkCount, client, From 16750fb19eae33e3d450722f2d3769277e024dbc Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 16:39:32 +0800 Subject: [PATCH 12/16] fix: remap demo citations to materialized document IDs After demo materialization, source rows get new materialized document IDs but the seeded demo-thread citations still reference the canonical IDs. Wire up the existing replaceDemoCitationDocumentId to remap citations after materialization completes, so source citation resolution continues to work. --- src/app/api/demo-sources/materialize/route.ts | 69 +++++++++++++++++++ src/domains/chat/chat-citation-persistence.ts | 25 ++++--- src/domains/chat/chat-message-repository.ts | 19 +++++ src/domains/chat/chat-thread-repository.ts | 25 +++++++ 4 files changed, 129 insertions(+), 9 deletions(-) diff --git a/src/app/api/demo-sources/materialize/route.ts b/src/app/api/demo-sources/materialize/route.ts index 7304ada..8e6f4b6 100644 --- a/src/app/api/demo-sources/materialize/route.ts +++ b/src/app/api/demo-sources/materialize/route.ts @@ -1,6 +1,11 @@ import { Effect } from "effect" import type { NextResponse } from "next/server" +import { chatCitationPersistence } from "@/domains/chat/chat-citation-persistence" +import { chatMessageRepository } from "@/domains/chat/chat-message-repository" +import { chatThreadRepository } from "@/domains/chat/chat-thread-repository" +import type { ChatCitationView } from "@/domains/chat/types" +import { databaseRuntime } from "@/domains/workspace/database-runtime" import { sourceService } from "@/domains/sources/service" import { toSourceView } from "@/domains/sources/view" import { notebookRequestContext } from "@/domains/workspace/request-context" @@ -75,6 +80,13 @@ export async function POST(request: Request): Promise { { concurrency: "unbounded" }, ) + // After materialization, remap seeded demo-thread citations from their + // canonical document IDs to the new materialized document IDs so source + // citation resolution continues to work. + yield* Effect.tryPromise(() => + fixDemoThreadCitations(workspace.id, materializedSources), + ).pipe(Effect.catchAllCause(() => Effect.void)) + return nextRouteResponse.toNextResponse(routeResult.ok({ sources })) }).pipe( Effect.catchAll(() => @@ -104,3 +116,60 @@ function getDemoSourceIds(value: unknown): string[] { function isRecord(value: unknown): value is Readonly> { return typeof value === "object" && value !== null } + +const seededDemoChatKey = "knowhere-demo-chat" + +async function fixDemoThreadCitations( + workspaceId: string, + materializedSources: ReadonlyArray<{ + readonly demoSourceId: string + readonly documentId: string + }>, +): Promise { + const catalog = await knowhereDemoApi.fetchCatalog() + const canonicalIdByDemoSourceId = new Map( + catalog.sources.map((s) => [s.demoSourceId, s.canonicalDocumentId]), + ) + const documentIdMap = new Map() + for (const source of materializedSources) { + const canonical = canonicalIdByDemoSourceId.get(source.demoSourceId) + if (canonical) { + documentIdMap.set(canonical, source.documentId) + } + } + if (documentIdMap.size === 0) return + + const thread = await databaseRuntime.runPromise( + chatThreadRepository.findThreadByDemoKeyEffect( + workspaceId, + seededDemoChatKey, + ), + ) + if (!thread) return + + const messages = await databaseRuntime.runPromise( + chatMessageRepository.listMessagesForThreadEffect(workspaceId, thread.id), + ) + if (!messages || messages.length === 0) return + + await Promise.all( + messages.map(async (message) => { + const currentCitations = message.citations as + | ChatCitationView[] + | null + | undefined + const updated = chatCitationPersistence.replaceDemoCitationDocumentId( + currentCitations ?? undefined, + documentIdMap, + ) + if (!updated) return + + await databaseRuntime.runPromise( + chatMessageRepository.updateMessageCitationsEffect( + message.id, + chatCitationPersistence.normalizeCitations(updated), + ), + ) + }), + ) +} diff --git a/src/domains/chat/chat-citation-persistence.ts b/src/domains/chat/chat-citation-persistence.ts index 498bcc1..a705362 100644 --- a/src/domains/chat/chat-citation-persistence.ts +++ b/src/domains/chat/chat-citation-persistence.ts @@ -13,7 +13,7 @@ type ChatCitationPersistence = { ) => CitationView[] | null readonly replaceDemoCitationDocumentId: ( citations: readonly ChatCitationView[] | undefined, - documentId: string, + documentIdMap: ReadonlyMap, ) => ChatCitationView[] | undefined } @@ -29,17 +29,24 @@ function normalizeCitations( function replaceDemoCitationDocumentId( citations: readonly ChatCitationView[] | undefined, - documentId: string, + documentIdMap: ReadonlyMap, ): ChatCitationView[] | undefined { if (!citations) return undefined - return citations.map((citation) => ({ - ...citation, - source: { - ...citation.source, - documentId, - }, - })) + return citations.map((citation) => { + const newId = citation.source.documentId + ? documentIdMap.get(citation.source.documentId) + : undefined + if (!newId) return citation + + return { + ...citation, + source: { + ...citation.source, + documentId: newId, + }, + } + }) } function toCitationView( diff --git a/src/domains/chat/chat-message-repository.ts b/src/domains/chat/chat-message-repository.ts index 66efa1c..4cc04f1 100644 --- a/src/domains/chat/chat-message-repository.ts +++ b/src/domains/chat/chat-message-repository.ts @@ -32,6 +32,10 @@ type ChatMessageRepository = { workspaceId: string, input: AppendChatMessageInput, ) => Effect.Effect + readonly updateMessageCitationsEffect: ( + messageId: string, + citations: CitationView[] | null, + ) => Effect.Effect } const listMessagesForThreadEffect: ChatMessageRepository["listMessagesForThreadEffect"] = @@ -92,7 +96,22 @@ const appendMessageToThreadEffect: ChatMessageRepository["appendMessageToThreadE ) }) +const updateMessageCitationsEffect: ChatMessageRepository["updateMessageCitationsEffect"] = + (messageId: string, citations: CitationView[] | null) => + Effect.gen(function* () { + const db = yield* DbClient + const [updated] = yield* Effect.promise(() => + db + .update(chatMessages) + .set({ citations }) + .where(eq(chatMessages.id, messageId)) + .returning(), + ) + return updated ?? null + }) + export const chatMessageRepository: ChatMessageRepository = { listMessagesForThreadEffect, appendMessageToThreadEffect, + updateMessageCitationsEffect, } diff --git a/src/domains/chat/chat-thread-repository.ts b/src/domains/chat/chat-thread-repository.ts index 7c6f9d1..d7cc884 100644 --- a/src/domains/chat/chat-thread-repository.ts +++ b/src/domains/chat/chat-thread-repository.ts @@ -52,6 +52,10 @@ type ChatThreadRepository = { workspaceId: string, threadId: string, ) => Effect.Effect + readonly findThreadByDemoKeyEffect: ( + workspaceId: string, + demoKey: string, + ) => Effect.Effect } const chatThreadListLimit = 50 @@ -249,6 +253,26 @@ const softDeleteThreadEffect: ChatThreadRepository["softDeleteThreadEffect"] = ( return result.length > 0 }) +const findThreadByDemoKeyEffect: ChatThreadRepository["findThreadByDemoKeyEffect"] = + (workspaceId: string, demoKey: string) => + Effect.gen(function* () { + const db = yield* DbClient + const row = yield* Effect.promise(() => + db + .select() + .from(chatThreads) + .where( + and( + eq(chatThreads.workspaceId, workspaceId), + eq(chatThreads.demoKey, demoKey), + isNull(chatThreads.deletedAt), + ), + ) + .limit(1), + ) + return row[0] ?? null + }) + export const chatThreadRepository: ChatThreadRepository = { findThreadInWorkspaceEffect, listThreadsForWorkspaceEffect, @@ -256,4 +280,5 @@ export const chatThreadRepository: ChatThreadRepository = { ensureDefaultThreadEffect, ensureDemoThreadEffect, softDeleteThreadEffect, + findThreadByDemoKeyEffect, } From b07c90be87188ac095e36f2b51e6b1c05d922024 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 17:26:47 +0800 Subject: [PATCH 13/16] fix: make demo citation chunks use materialized document IDs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes to complete the citation fix after demo materialization: 1. route-chunks: when serving chunks for a materialized demo source, use the source's knowhereDocumentId as the chunk documentId instead of the canonical one, so chunk resolution finds matching citations. 2. workspace-chat-workflow: after materialization, re-fetch the chat thread from the server instead of remapping citation documentIds client-side. The server-side fix (prior commit) already persisted the corrected citations; this just gets fresh data to the client. Also hide the materialization status text from the user — it now shows "Thinking" like a normal send. --- src/components/workspace-chat-workflow.ts | 15 ++++++++++++++- src/domains/sources/route-chunks.ts | 4 +++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/components/workspace-chat-workflow.ts b/src/components/workspace-chat-workflow.ts index 63ad9eb..d50747f 100644 --- a/src/components/workspace-chat-workflow.ts +++ b/src/components/workspace-chat-workflow.ts @@ -218,12 +218,25 @@ export function useWorkspaceChatWorkflow({ const demoSourceIds = getMaterializableDemoSourceIds(sources) if (demoSourceIds.length > 0) { setChat((current) => - workspaceChatState.prepareSend(current, "Preparing demo sources..."), + workspaceChatState.prepareSend(current, "Thinking"), ) try { const materializedSources = await workspaceClient.materializeDemoSources({ demoSourceIds }) onSourcesMaterialized?.(demoSourceIds, materializedSources) + if (chat.threadId) { + try { + const fresh = await workspaceClient.fetchChatThread(chat.threadId) + setChat((current) => { + if (current.threadId !== fresh.requestedThreadId) return current + if (!fresh.thread || !Array.isArray(fresh.messages)) + return current + return { ...current, messages: [...fresh.messages] } + }) + } catch { + // stale citations until page reload — materialization succeeded + } + } } catch { setChat((current) => ({ ...current, diff --git a/src/domains/sources/route-chunks.ts b/src/domains/sources/route-chunks.ts index 6404041..60ba8b1 100644 --- a/src/domains/sources/route-chunks.ts +++ b/src/domains/sources/route-chunks.ts @@ -69,6 +69,7 @@ const loadSourceChunksEffect = ( input, deps, source.demoKey, + source.knowhereDocumentId, ) return demoResult ?? sourceNotFound() } @@ -100,6 +101,7 @@ const loadDemoChunkPageEffect = ( input: LoadSourceChunksInput, deps: RouteChunksDependencies, demoSourceId: string = input.sourceId, + documentIdOverride?: string | null, ) => Effect.gen(function* () { const pages = input.shouldLoadAll @@ -124,7 +126,7 @@ const loadDemoChunkPageEffect = ( title: page.title, mimeType: page.mimeType, status: "ready" as const, - documentId: page.canonicalDocumentId, + documentId: documentIdOverride ?? page.canonicalDocumentId, } const chunks = pages.flatMap((demoChunkPage) => demoChunkPage.chunks.map((chunk) => From fa83adc998c2b434608fd480bcc0993ccd86c5b4 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 17:44:03 +0800 Subject: [PATCH 14/16] perf: skip workspace DB queries for non-UUID demo source chunk requests Non-UUID source IDs can only be demo sources. Short-circuit loadSourceChunksEffect before getCurrentUser/ensureWorkspace to avoid unnecessary DB queries on every demo chunk page load. Also caches the upstream demo chunk API response for 5 minutes. --- .../sources/[sourceId]/chunks/route.test.ts | 19 ++++++++----------- src/domains/sources/route-chunks.ts | 16 +++++++++++++++- src/integrations/knowhere-demo.ts | 2 +- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/app/api/sources/[sourceId]/chunks/route.test.ts b/src/app/api/sources/[sourceId]/chunks/route.test.ts index 3a7adb3..4b7ebf7 100644 --- a/src/app/api/sources/[sourceId]/chunks/route.test.ts +++ b/src/app/api/sources/[sourceId]/chunks/route.test.ts @@ -266,10 +266,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { page: 1, pageSize: 100, }) - expect(mocks.findSourceInWorkspace).toHaveBeenCalledWith( - "workspace_1", - "demo-tsla-q4-2025", - ) + expect(mocks.findSourceInWorkspace).not.toHaveBeenCalled() expect(mocks.ensureApiKeyForWorkspace).not.toHaveBeenCalled() expect(mocks.makeKnowhereClient).not.toHaveBeenCalled() expect(mocks.getSourceParseAssetUrls).not.toHaveBeenCalled() @@ -288,7 +285,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { createdAt: new Date("2026-05-10T00:00:00.000Z"), }) mocks.findSourceInWorkspace.mockResolvedValue({ - id: "source_materialized_demo", + id: "00000000-0000-0000-0000-000000000001", workspaceId: "workspace_1", title: "TSLA-Q4-2025-Update.pdf", mimeType: "application/pdf", @@ -335,16 +332,16 @@ describe("GET /api/sources/[sourceId]/chunks", () => { const response = await GET( new NextRequest( - "http://localhost:3001/api/sources/source_materialized_demo/chunks?page=1&pageSize=100", + "http://localhost:3001/api/sources/00000000-0000-0000-0000-000000000001/chunks?page=1&pageSize=100", ), - { params: Promise.resolve({ sourceId: "source_materialized_demo" }) }, + { params: Promise.resolve({ sourceId: "00000000-0000-0000-0000-000000000001" }) }, ) await expect(response.json()).resolves.toMatchObject({ chunks: [ { chunkId: "demo-tsla-q4-2025:chunk_1", - documentId: "demo-doc-tsla-q4-2025", + documentId: "copied-doc-tsla-q4-2025", sourceTitle: "TSLA-Q4-2025-Update.pdf", }, ], @@ -452,7 +449,7 @@ describe("GET /api/sources/[sourceId]/chunks", () => { createdAt: new Date("2026-05-10T00:00:00.000Z"), }) mocks.findSourceInWorkspace.mockResolvedValue({ - id: "source_1", + id: "00000000-0000-0000-0000-000000000002", workspaceId: "workspace_1", title: "notes.pdf", mimeType: "application/pdf", @@ -476,9 +473,9 @@ describe("GET /api/sources/[sourceId]/chunks", () => { const response = await GET( new NextRequest( - "http://localhost:3001/api/sources/source_1/chunks?page=1&pageSize=1", + "http://localhost:3001/api/sources/00000000-0000-0000-0000-000000000002/chunks?page=1&pageSize=1", ), - { params: Promise.resolve({ sourceId: "source_1" }) }, + { params: Promise.resolve({ sourceId: "00000000-0000-0000-0000-000000000002" }) }, ) await expect(response.json()).resolves.toMatchObject({ diff --git a/src/domains/sources/route-chunks.ts b/src/domains/sources/route-chunks.ts index 60ba8b1..f85ad34 100644 --- a/src/domains/sources/route-chunks.ts +++ b/src/domains/sources/route-chunks.ts @@ -5,6 +5,7 @@ import type { DemoChunkPage } from "@/integrations/knowhere-demo" import { logger } from "@/lib/logger" import { routeResult } from "@/lib/route-result" import { getClientForWorkspace } from "./route-dependencies" +import { sourceRowRepository } from "./source-row-repository" import type { JsonRouteResult, LoadSourceChunksInput, @@ -46,6 +47,11 @@ const loadSourceChunksEffect = ( deps: RouteChunksDependencies, ) => Effect.gen(function* () { + if (!sourceRowRepository.isWorkspaceSourceId(input.sourceId)) { + const demoResult = yield* loadDemoChunkPageEffect(input, deps) + return demoResult ?? sourceNotFound() + } + const user = yield* Effect.tryPromise(() => deps.getCurrentUser()) if (!user) { const demoResult = yield* loadDemoChunkPageEffect(input, deps) @@ -152,7 +158,7 @@ const loadDemoChunkPageEffect = ( pageSize: input.pageParams.pageSize, shouldLoadAll: input.shouldLoadAll, knowhereBaseUrl: process.env.KNOWHERE_BASE_URL ?? "(default)", - error: error instanceof Error ? error.message : String(error), + error: getErrorMessage(error), }) return null }), @@ -187,6 +193,14 @@ async function loadAllDemoChunkPages( return pages } +function getErrorMessage(error: unknown): string { + if (error instanceof Error) { + const inner = (error as Error & { error?: unknown }).error + return inner instanceof Error ? inner.message : error.message + } + return String(error) +} + function sourceNotFound(): JsonRouteResult<{ readonly message: string }> { return routeResult.error(404, "Source not found.") } diff --git a/src/integrations/knowhere-demo.ts b/src/integrations/knowhere-demo.ts index b10434a..37512ca 100644 --- a/src/integrations/knowhere-demo.ts +++ b/src/integrations/knowhere-demo.ts @@ -215,7 +215,7 @@ const fetchChunkPageEffect = Effect.fn("knowhereDemo.fetchChunkPage")( resolveApiURL( `/api/v1/demo/sources/${encodeURIComponent(input.demoSourceId)}/chunks?${params.toString()}`, ), - { cache: "no-store" }, + { cache: "force-cache", next: { revalidate: 300 } }, ), ) yield* assertOkEffect(response) From 068431831d8b860efc8e325f8b65756ed6d72320 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 17:44:33 +0800 Subject: [PATCH 15/16] fix: update schema path in drizzle config and add Upstash CLI command --- drizzle.config.ts | 2 +- package.json | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/drizzle.config.ts b/drizzle.config.ts index 7148eb7..03d9850 100644 --- a/drizzle.config.ts +++ b/drizzle.config.ts @@ -9,7 +9,7 @@ import { defineConfig } from "drizzle-kit"; * pnpm db:migrate # apply migrations to DATABASE_URL (prod deploy) */ export default defineConfig({ - schema: "./src/lib/schema.ts", + schema: "./src/infrastructure/db/schema.ts", out: "./drizzle", dialect: "postgresql", dbCredentials: { diff --git a/package.json b/package.json index 3a13bf2..6570d03 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "db:generate": "drizzle-kit generate", "db:push": "drizzle-kit push", "db:migrate": "drizzle-kit migrate", - "db:studio": "drizzle-kit studio" + "db:studio": "drizzle-kit studio", + "upstash:dev": "npx @upstash/qstash-cli dev" }, "dependencies": { "@ai-sdk/react": "^3.0.177", @@ -80,4 +81,4 @@ "typescript": "^6.0.3", "vitest": "^4.1.5" } -} +} \ No newline at end of file From f33430b03e5e2e84551bbe3194be8b70f523bec8 Mon Sep 17 00:00:00 2001 From: suguanYang Date: Wed, 13 May 2026 17:52:17 +0800 Subject: [PATCH 16/16] fix: remove chunk hash link and dashboard link icon Drop the #chunk- URL hash navigation and the LayoutDashboard icon prefix from the top-nav dashboard link. --- src/components/top-nav.tsx | 3 +- src/components/workspace-citation-focus.ts | 17 +------- src/lib/use-hash-fragment.test.ts | 17 -------- src/lib/use-hash-fragment.ts | 45 ---------------------- 4 files changed, 3 insertions(+), 79 deletions(-) delete mode 100644 src/lib/use-hash-fragment.test.ts delete mode 100644 src/lib/use-hash-fragment.ts diff --git a/src/components/top-nav.tsx b/src/components/top-nav.tsx index c0fbab5..487fcc2 100644 --- a/src/components/top-nav.tsx +++ b/src/components/top-nav.tsx @@ -1,7 +1,7 @@ import { NotebookLogoMark } from "@/components/notebook-logo-mark"; import { Separator } from "@/components/ui/separator"; import { ThemeToggle } from "@/components/theme-toggle"; -import { ExternalLink, LayoutDashboard } from "lucide-react"; +import { ExternalLink } from "lucide-react"; import type { ReactElement } from "react"; export type TopNavProps = { @@ -41,7 +41,6 @@ export function TopNav({ aria-label="Open Dashboard" className="flex items-center gap-1.5 rounded-md px-2 py-1.5 text-xs font-medium text-muted-foreground transition-colors hover:text-foreground hover:underline focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-[#8e51ff]/25" > - Dashboard diff --git a/src/components/workspace-citation-focus.ts b/src/components/workspace-citation-focus.ts index 7ca7a0f..d428bf0 100644 --- a/src/components/workspace-citation-focus.ts +++ b/src/components/workspace-citation-focus.ts @@ -1,10 +1,9 @@ "use client" -import { useCallback, useEffect, useState } from "react" +import { useCallback, useState } from "react" import { workspaceCitationState } from "@/components/workspace-citation-state" import { useWorkspaceSelectedChunks } from "@/components/workspace-selected-chunks" -import { useHashFragment } from "@/lib/use-hash-fragment" import type { ChatCitationView } from "@/domains/chat/types" import type { ParsedChunkView } from "@/domains/chunks/types" import type { SourceView } from "@/domains/sources/types" @@ -58,7 +57,6 @@ export function useWorkspaceCitationFocus({ ) const [prefetchedChunksBySourceId, setPrefetchedChunksBySourceId] = useState(initialPrefetchedChunksBySourceId) - const [hashChunkId, setHashChunkId] = useHashFragment() const { hasMoreSelectedChunks, handleLoadMoreChunks, @@ -78,21 +76,10 @@ export function useWorkspaceCitationFocus({ chunkId, requestId: current.requestId + 1, })) - setHashChunkId(chunkId) }, - [setHashChunkId], + [], ) - useEffect(() => { - if (!hashChunkId) return - - const frameId = window.requestAnimationFrame(() => { - requestChunkFocus(hashChunkId) - }) - - return () => window.cancelAnimationFrame(frameId) - }, [hashChunkId, requestChunkFocus]) - const handleSourceSelected = useCallback( (sourceId: string | null): void => { onSelectSource(sourceId) diff --git a/src/lib/use-hash-fragment.test.ts b/src/lib/use-hash-fragment.test.ts deleted file mode 100644 index 91ddda4..0000000 --- a/src/lib/use-hash-fragment.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -// @vitest-environment node -import React from "react"; -import { renderToString } from "react-dom/server"; -import { describe, expect, it } from "vitest"; - -import { useHashFragment } from "./use-hash-fragment"; - -describe("useHashFragment", () => { - it("can render on the server without reading window", () => { - function HashFragmentProbe(): React.ReactElement { - const [chunkId] = useHashFragment(); - return React.createElement("span", null, chunkId ?? "none"); - } - - expect(() => renderToString(React.createElement(HashFragmentProbe))).not.toThrow(); - }); -}); diff --git a/src/lib/use-hash-fragment.ts b/src/lib/use-hash-fragment.ts deleted file mode 100644 index 0ab9ddd..0000000 --- a/src/lib/use-hash-fragment.ts +++ /dev/null @@ -1,45 +0,0 @@ -"use client"; - -import { useCallback, useEffect, useState } from "react"; - -const CHUNK_PREFIX = "#chunk-"; - -export function useHashFragment(): [ - chunkId: string | null, - navigateToChunk: (id: string | null) => void, -] { - const [chunkId, setChunkId] = useState(() => - typeof window === "undefined" ? null : readHash(window.location.hash), - ); - - useEffect(() => { - function onHashChange(): void { - setChunkId(readHash(window.location.hash)); - } - onHashChange(); - window.addEventListener("hashchange", onHashChange); - return () => window.removeEventListener("hashchange", onHashChange); - }, []); - - const navigateToChunk = useCallback((id: string | null) => { - if (id) { - window.location.hash = `${CHUNK_PREFIX}${id}`; - } else { - window.history.replaceState( - null, - "", - window.location.pathname + window.location.search, - ); - setChunkId(null); - } - }, []); - - return [chunkId, navigateToChunk]; -} - -function readHash(hash: string): string | null { - if (hash.startsWith(CHUNK_PREFIX)) { - return hash.slice(CHUNK_PREFIX.length); - } - return null; -}