fix: add dev API key support and demo flow improvements

.env.local.example

@@ -2,6 +2,11 @@
 # Use staging when validating staging keys.
 # KNOWHERE_BASE_URL=https://api-staging.knowhereto.ai
 
+# Optional development override. When set, Notebook skips Dashboard session
+# auth and Dashboard-issued JWT creation, then calls Knowhere directly with
+# this key. Leave unset for production and Dashboard-authenticated staging.
+# KNOWHERE_API_KEY=sk_your_development_key_here
+
 # --- Chat provider (server-side only) ---
 # Vercel AI Gateway key; AI SDK picks it up automatically
 AI_GATEWAY_API_KEY=vck_your_key_here

README.md

@@ -11,6 +11,7 @@ Upload documents, explore parsed content, and ask questions about your knowledge
 
 2. Fill in your API keys in `.env.local`:
    - `AI_GATEWAY_API_KEY` — your Vercel AI Gateway key for chat (optional `CHAT_MODEL` override)
+   - `KNOWHERE_API_KEY` — optional development override that skips Dashboard auth and calls Knowhere directly
 
 3. Install dependencies and run:
    ```bash
@@ -42,6 +43,12 @@ Notebook treats Dashboard as the auth source of truth. Server-side auth calls
 forward the incoming session cookie to Dashboard oRPC endpoints, including
 `/api/orpc/users/getCurrentUser` and `/api/orpc/users/issueServiceJwt`.
 
+For local development, setting server-side `KNOWHERE_API_KEY` switches Notebook
+into API-key mode. In that mode the app uses a deterministic local development
+user, skips Dashboard redirects and JWT issuance, and passes the configured key
+directly to the Knowhere SDK. Leave it unset for production and normal
+Dashboard-authenticated staging flows.
+
 Dashboard chooses its oRPC handler by request shape and `Content-Type`.
 When using Effect's `HttpClientRequest.bodyText`, pass
 `"application/json"` as the body content type. Setting the header before

drizzle.config.ts

@@ -9,7 +9,7 @@ import { defineConfig } from "drizzle-kit";
  *   pnpm db:migrate    # apply migrations to DATABASE_URL (prod deploy)
  */
 export default defineConfig({
-  schema: "./src/lib/schema.ts",
+  schema: "./src/infrastructure/db/schema.ts",
   out: "./drizzle",
   dialect: "postgresql",
   dbCredentials: {

package.json

@@ -16,7 +16,8 @@
     "db:generate": "drizzle-kit generate",
     "db:push": "drizzle-kit push",
     "db:migrate": "drizzle-kit migrate",
-    "db:studio": "drizzle-kit studio"
+    "db:studio": "drizzle-kit studio",
+    "upstash:dev": "npx @upstash/qstash-cli dev"
   },
   "dependencies": {
     "@ai-sdk/react": "^3.0.177",
@@ -34,6 +35,7 @@
     "@radix-ui/react-tabs": "^1.1.13",
     "@radix-ui/react-tooltip": "^1.2.8",
     "@tanstack/react-virtual": "^3.13.24",
+    "@upstash/workflow": "^1.2.1",
     "@vercel/blob": "^2.3.3",
     "ai": "^6.0.175",
     "class-variance-authority": "^0.7.1",
@@ -79,4 +81,4 @@
     "typescript": "^6.0.3",
     "vitest": "^4.1.5"
   }
-}
+}

src/app/api/demo-sources/materialize/route.ts

@@ -1,5 +1,11 @@
+import { Effect } from "effect"
 import type { NextResponse } from "next/server"
 
+import { chatCitationPersistence } from "@/domains/chat/chat-citation-persistence"
+import { chatMessageRepository } from "@/domains/chat/chat-message-repository"
+import { chatThreadRepository } from "@/domains/chat/chat-thread-repository"
+import type { ChatCitationView } from "@/domains/chat/types"
+import { databaseRuntime } from "@/domains/workspace/database-runtime"
 import { sourceService } from "@/domains/sources/service"
 import { toSourceView } from "@/domains/sources/view"
 import { notebookRequestContext } from "@/domains/workspace/request-context"
@@ -8,76 +14,162 @@ import { nextRouteResponse } from "@/lib/next-route-response"
 import { routeResult } from "@/lib/route-result"
 
 export async function POST(request: Request): Promise<NextResponse> {
-  const body = await routeResult.readJson(request)
-  if (!body.ok) {
-    return nextRouteResponse.toNextResponse(
-      routeResult.badRequest("Invalid request body."),
-    )
-  }
+  return Effect.runPromise(
+    Effect.gen(function* () {
+      const body = yield* Effect.tryPromise(() =>
+        routeResult.readJson(request),
+      )
+      if (!body.ok) {
+        return nextRouteResponse.toNextResponse(
+          routeResult.badRequest("Invalid request body."),
+        )
+      }
 
-  const demoSourceIds = getDemoSourceIds(body.value)
-  if (demoSourceIds.length === 0) {
-    return nextRouteResponse.toNextResponse(
-      routeResult.badRequest("Select at least one demo source."),
-    )
-  }
+      const demoSourceIds = getDemoSourceIds(body.value)
+      if (demoSourceIds.length === 0) {
+        return nextRouteResponse.toNextResponse(
+          routeResult.badRequest("Select at least one demo source."),
+        )
+      }
 
-  try {
-    const { apiKey, workspace } =
-      await notebookRequestContext.getAuthenticatedWithClient()
-    const hiddenDemoSourceIds = new Set(
-      await sourceService.listHiddenDemoSourceIds(workspace.id),
-    )
-    const visibleDemoSourceIds = demoSourceIds.filter(
-      (demoSourceId) => !hiddenDemoSourceIds.has(demoSourceId),
-    )
-    if (visibleDemoSourceIds.length === 0) {
-      return nextRouteResponse.toNextResponse(
-        routeResult.badRequest("Selected demo sources are no longer available."),
+      const { apiKey, workspace } = yield* Effect.tryPromise(() =>
+        notebookRequestContext.getAuthenticatedWithClient(),
       )
-    }
-
-    const materializedSources = await knowhereDemoApi.materializeSources({
-      apiKey,
-      namespace: workspace.namespace,
-      demoSourceIds: visibleDemoSourceIds,
-    })
-    const sources = await Promise.all(
-      materializedSources.map(async (source) => {
-        const row = await sourceService.upsertMaterializedDemoSource(
-          workspace.id,
-          {
-            demoSourceId: source.demoSourceId,
-            title: source.title,
-            mimeType: source.mimeType,
-            sizeBytes: source.sizeBytes,
-            knowhereDocumentId: source.documentId,
-            originalBlobUrl: `/api/demo-sources/${encodeURIComponent(
-              source.demoSourceId,
-            )}/original`,
-          },
+      const hiddenDemoSourceIds = new Set(
+        yield* Effect.tryPromise(() =>
+          sourceService.listHiddenDemoSourceIds(workspace.id),
+        ),
+      )
+      const visibleDemoSourceIds = demoSourceIds.filter(
+        (demoSourceId) => !hiddenDemoSourceIds.has(demoSourceId),
+      )
+      if (visibleDemoSourceIds.length === 0) {
+        return nextRouteResponse.toNextResponse(
+          routeResult.badRequest(
+            "Selected demo sources are no longer available.",
+          ),
         )
-        return toSourceView(row, { chunkCount: source.chunkCount })
-      }),
-    )
-
-    return nextRouteResponse.toNextResponse(routeResult.ok({ sources }))
-  } catch {
-    return nextRouteResponse.toNextResponse(
-      routeResult.error(502, "Demo sources could not be prepared right now."),
-    )
-  }
+      }
+
+      const materializedSources = yield* Effect.tryPromise(() =>
+        knowhereDemoApi.materializeSources({
+          apiKey,
+          namespace: workspace.namespace,
+          demoSourceIds: visibleDemoSourceIds,
+        }),
+      )
+
+      const sources = yield* Effect.all(
+        materializedSources.map((source) =>
+          Effect.gen(function* () {
+            const row = yield* Effect.tryPromise(() =>
+              sourceService.upsertMaterializedDemoSource(workspace.id, {
+                demoSourceId: source.demoSourceId,
+                title: source.title,
+                mimeType: source.mimeType,
+                sizeBytes: source.sizeBytes,
+                knowhereDocumentId: source.documentId,
+                originalBlobUrl: `/api/demo-sources/${encodeURIComponent(
+                  source.demoSourceId,
+                )}/original`,
+              }),
+            )
+            return toSourceView(row, { chunkCount: source.chunkCount })
+          }),
+        ),
+        { concurrency: "unbounded" },
+      )
+
+      // After materialization, remap seeded demo-thread citations from their
+      // canonical document IDs to the new materialized document IDs so source
+      // citation resolution continues to work.
+      yield* Effect.tryPromise(() =>
+        fixDemoThreadCitations(workspace.id, materializedSources),
+      ).pipe(Effect.catchAllCause(() => Effect.void))
+
+      return nextRouteResponse.toNextResponse(routeResult.ok({ sources }))
+    }).pipe(
+      Effect.catchAll(() =>
+        Effect.succeed(
+          nextRouteResponse.toNextResponse(
+            routeResult.error(
+              502,
+              "Demo sources could not be prepared right now.",
+            ),
+          ),
+        ),
+      ),
+    ),
+  )
 }
 
 function getDemoSourceIds(value: unknown): string[] {
   if (!isRecord(value) || !Array.isArray(value.demoSourceIds)) return []
 
   const selectedIds = value.demoSourceIds.filter(
-    (item): item is string => typeof item === "string" && item.trim().length > 0,
+    (item): item is string =>
+      typeof item === "string" && item.trim().length > 0,
   )
   return Array.from(new Set(selectedIds.map((item) => item.trim())))
 }
 
 function isRecord(value: unknown): value is Readonly<Record<string, unknown>> {
   return typeof value === "object" && value !== null
 }
+
+const seededDemoChatKey = "knowhere-demo-chat"
+
+async function fixDemoThreadCitations(
+  workspaceId: string,
+  materializedSources: ReadonlyArray<{
+    readonly demoSourceId: string
+    readonly documentId: string
+  }>,
+): Promise<void> {
+  const catalog = await knowhereDemoApi.fetchCatalog()
+  const canonicalIdByDemoSourceId = new Map(
+    catalog.sources.map((s) => [s.demoSourceId, s.canonicalDocumentId]),
+  )
+  const documentIdMap = new Map<string, string>()
+  for (const source of materializedSources) {
+    const canonical = canonicalIdByDemoSourceId.get(source.demoSourceId)
+    if (canonical) {
+      documentIdMap.set(canonical, source.documentId)
+    }
+  }
+  if (documentIdMap.size === 0) return
+
+  const thread = await databaseRuntime.runPromise(
+    chatThreadRepository.findThreadByDemoKeyEffect(
+      workspaceId,
+      seededDemoChatKey,
+    ),
+  )
+  if (!thread) return
+
+  const messages = await databaseRuntime.runPromise(
+    chatMessageRepository.listMessagesForThreadEffect(workspaceId, thread.id),
+  )
+  if (!messages || messages.length === 0) return
+
+  await Promise.all(
+    messages.map(async (message) => {
+      const currentCitations = message.citations as
+        | ChatCitationView[]
+        | null
+        | undefined
+      const updated = chatCitationPersistence.replaceDemoCitationDocumentId(
+        currentCitations ?? undefined,
+        documentIdMap,
+      )
+      if (!updated) return
+
+      await databaseRuntime.runPromise(
+        chatMessageRepository.updateMessageCitationsEffect(
+          message.id,
+          chatCitationPersistence.normalizeCitations(updated),
+        ),
+      )
+    }),
+  )
+}