diff --git a/.gitignore b/.gitignore index cd698f84..53b52c7a 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,12 @@ test-results/ /[0-9][0-9]-*.jpg /[0-9][0-9]-*.yml +# Audit subdirectory at repo root — agents bucket their per-session +# screenshots under `audit//` so the root-level +# `/audit-*.png` patterns above don't catch them. Anchor the dir +# itself so the whole tree is ignored. +/audit/ + # macOS Finder duplicate files + directories (caught by hygiene CI; should # never reach repo). Cover both extension-bearing files (`Foo 2.tsx`) and # extension-less files (`pre-push 2`, `.npmrc 2`) and dup-named dirs @@ -81,3 +87,8 @@ test-results/ " 2".* " 3".* .vercel + +# Local Playwright snapshot artifacts (never commit) +workspace-snapshot.md +.playwright-mcp/ +.env*.local diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 00000000..9d27dc84 --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,26 @@ +# gitleaks per-commit allowlist +# https://github.com/gitleaks/gitleaks#gitleaksignore +# +# All entries below are findings in HISTORICAL commits that are no +# longer reachable from any branch HEAD after the 2026-05-15 BFG +# history scrub (see SECURITY-INCIDENT-2026-05-14.md for the +# incident write-up). They remain reachable only via the +# `gitleaks-pre-scrub-2026-05-15-rollback` tag, which is the +# emergency-rollback safety belt and will be deleted ~7 days after +# the scrub once production has burned in cleanly. +# +# The findings are test stubs — fake keys shaped like the Voyage AI +# `pa-` prefix but with literal fixture values like +# `pa-test-key-1234567890`. Inline `// gitleaks:allow` annotations +# have been added to the live versions of those test files, so the +# fingerprints below stop being findings once the rollback tag is +# deleted. + +# voyage-client.test.ts (line 18 of commit 080b66b0) — test stub +080b66b0262dd6ef68775547873747bf3653b913:apps/web/tests/unit/ai/voyage-client.test.ts:generic-api-key:18 + +# semantic-search-tool.test.ts (line 40 of commit 080b66b0) — test stub +080b66b0262dd6ef68775547873747bf3653b913:apps/web/tests/unit/ai/semantic-search-tool.test.ts:generic-api-key:40 + +# semantic-search-tool.test.ts (line 96 of commit ae20dd72) — test stub +ae20dd7245310a1a4694db9f2657a70e4f2b1353:apps/web/tests/unit/ai/semantic-search-tool.test.ts:generic-api-key:96 diff --git a/CLAUDE.md b/CLAUDE.md index 7222d755..54da7d28 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,9 +2,94 @@ Agent context for the unified NDI Cloud monorepo at `ndi-cloud.com`. +--- + +## 🚨 ORIENTATION — READ THIS FIRST (every session) + +You are working across **two sibling repos** under `~/Documents/ndi-projects/`: + +| Repo | Path | Role | Hosted on | +|---|---|---|---| +| `ndi-cloud-app` | `~/Documents/ndi-projects/ndi-cloud-app` | Next.js 16 frontend + API routes | Vercel | +| `ndi-data-browser-v2` | `~/Documents/ndi-projects/ndi-data-browser-v2` | FastAPI backend + NDI-python integration | Railway | + +**Active branches:** + +| Repo | `main` | Draft branch (where we work) | +|---|---|---| +| `ndi-cloud-app` | production — **DO NOT push** | `feat/experimental-ask-chat` | +| `ndi-data-browser-v2` | production — **DO NOT push** | `feat/ndi-python-phase-a` | + +### THE LIVE DEPLOYMENT IS SACRED — DO NOT TOUCH + +| | Production (untouched) | Experimental / Preview (where we work) | +|---|---|---| +| **Frontend URL** | `https://ndi-cloud.com` | `https://ndi-cloud-app-web-git-feat-experiment-c5da7d-ndi-cloud-a83eb4e7.vercel.app` | +| **Backend URL** | `https://ndb-v2-production.up.railway.app` | `https://ndb-v2-experimental.up.railway.app` | +| **Railway env id** | `e0c00fb7-ac98-431f-acdb-f4988032160f` | `90101f6e-042b-44d6-8c8d-ec18d43b341b` | +| **Vercel env scope** | `Production` | `Preview` | +| **Branch wired to** | `main` of each repo | the draft branches above | + +### Sacred rules (non-negotiable) + +1. **NEVER push to `main`** on either repo. +2. **NEVER touch Vercel `Production`-scope env vars.** Touch only `Preview`. +3. **NEVER touch Railway `production` env.** Touch only `experimental` (env id `90101f6e-...` for ndb-v2). The Railway agent lets you specify env id — always use the experimental one. +4. **NEVER force-push to `main`.** Force-push on the draft branch is OK if explicitly authorized. +5. **NEVER skip pre-commit / pre-push hooks** (`--no-verify`, `--no-gpg-sign` are prohibited). +6. **Author rule (non-negotiable):** every commit must be `audriB `. Use `--author="audriB "` on every git commit. +7. **Co-Authored-By trailer required** on every Claude-driven commit: `Co-Authored-By: Claude Opus 4.7 (1M context) `. + +### Test credentials (Playwright form-fill ONLY; never persist or echo) + +For workspace + chat smoke testing: +- email: `audri+test@walthamdatascience.com` +- password: `remhuz-ruwfy4-jiGcen` + +Deliberately-scoped test account. Public datasets only — no private datasets attached. Use Playwright `browser_fill_form`; never write to disk; never echo in chat output. + +### Verifying before any action + +```bash +# Confirm you're on the right branch +git branch --show-current +# cloud-app should print: feat/experimental-ask-chat +# ndb-v2 should print: feat/ndi-python-phase-a + +# Confirm Railway env id you're targeting (in railway-agent calls) +# experimental ndb-v2: 90101f6e-042b-44d6-8c8d-ec18d43b341b +# DO NOT use production: e0c00fb7-ac98-431f-acdb-f4988032160f +``` + +If you ever find yourself about to operate on `main` or on production Vercel/Railway, **STOP** and ask the user for explicit confirmation. + +### Where to read next (pick up cold) + +**Read this FIRST:** [`apps/web/docs/HANDOFF.md`](apps/web/docs/HANDOFF.md) +— single source of truth for current project state. Has branch verification +commands, sacred rules, test creds, the live deployment state, the +experimental branch + GitHub Template arc status, work-done + work-left, +and operational gotchas. Every prior `*-handoff*.md` / `*-pre-compact-*.md` +under `docs/reviews/` and `docs/specs/` is marked SUPERSEDED with a pointer +back to HANDOFF.md. + +Reference docs still canonical (read when their topic comes up): + +- `apps/web/docs/architecture/decisions/` — ADRs 001-010 +- `apps/web/docs/operations/` — workspace tutorial, disaster recovery, + HIPAA mapping, audit-log policy, the recent NDI-python + NDI-matlab + API audits, code-export coverage matrix, memory-crash investigation +- `apps/web/docs/operations/workspace-tutorial.md` — drives the G2/G3 + parity smoke + +Audit artifacts (gitignored, on-disk only — DO NOT try to commit them): +- `audit/2026-05-18-parity-and-tutorials/` — agent reports (E/F/G/G-verify/G2-stub/DB-DD-verify), screenshots from every Playwright session. + +--- + ## What this repo is -Next.js 15 App Router monorepo. Replaces: +Next.js 16 App Router monorepo. Replaces: - `Waltham-Data-Science/ndi-web-app-wds` (Pages Router marketing site) - `Waltham-Data-Science/ndi-data-browser-v2` frontend (Vite SPA + React Router) @@ -34,26 +119,94 @@ Phases that have landed (chronological, by lead PR): - PRs #147–155 — round-4 + round-5 team review polish (Steve's feedback): ontology Name-cell linkification, marketing copy without Crossref branding, dataset-DOI restructure with PMID/PMC pills, QuickPlot column-first redesign, SEO upgrades (Dataset JSON-LD, per-dataset sitemap), Griswold timeout bump, Cite modal copy + Download buttons, test-suite audit (+106 tests) - PR #156 — Phase 7 cleanup: restore strict apex-only Origin allowlist (drop pre-cutover hardcode + env-var escape hatch), shipped immediately post-swap -Reference plans: -- High-level: see Audri's plan file at `/Users/audribhowmick/.claude/plans/sharded-puzzling-dragonfly.md` -- Pre-cutover audit (this session): `/Users/audribhowmick/.claude/plans/atomic-sniffing-island.md` -- Architectural rationale: `ndi-data-browser-v2/docs/plans/cross-repo-unification-2026-04-24.md` +### Current draft branch in flight — `feat/experimental-ask-chat` (PR #160) + +**This branch is NOT on production.** It carries the experimental `/ask` chat + the workspace at `/my/workspace/[id]` + several Phase 8 polish items. It is paired with a separate Railway env (`ndb-v2-experimental`) running NDI-python integration Phase A. The branch-aware rewrite in `apps/web/next.config.ts` routes preview deploys of this branch to the experimental Railway env automatically. + +**Key in-flight work (post-2026-05-15, 94% of master plan landed):** +- `/ask` chat with 17 tools (psth, fetch_signal, fetch_image, fetch_spike_summary, treatment_timeline, tabular_query, query_documents, walk_provenance, ndi_query, ndi_dataset_overview, get_document, aggregate_documents, lookup_ontology, list_published_datasets, get_dataset, get_dataset_summary, get_dataset_class_counts, get_facets, semantic_search_datasets). Architecture: ADR-001 keeps the heart on Railway; ADR-002 puts every handler in `lib/ndi/tools/`; ADR-003 forwards auth via the optional `ToolContext`. **AI SDK is now v6** (`ai@6 @ai-sdk/anthropic@3 @ai-sdk/react@3`). +- **NEW auth-gated `/my/ask`** route reusing the same ``. Anonymous → redirect to /login. `canUseAsk === false` → "feature not enabled for your org" notice. The legacy `/(marketing)/ask` route stays live during the transition. +- Workspace at `/my/workspace/[id]/...` with 7 panels (DatasetStructure, BehavioralCompare, TreatmentTimeline, SignalViewer, PSTH, SpikeActivity, ElectrodePosition). Each panel ports a chat tool's chart_payload contract into a per-dataset UI. **All 7 canonicalized to `` chrome.** +- **Dataset Health:** invariants module at `lib/data-quality/invariants.ts` (6 invariants), nightly cron at `/api/cron/dataset-health` (07:23 UTC in vercel.json) writing to `dataset_health_violations` Postgres table, admin dashboard at `/admin/data-health`, catalog badge at `` on each `DatasetCard`. +- **Cost tracking:** `chat_usage_events` Postgres table; `lib/usage/rate-card.ts` + `lib/usage/log.ts` wired into `/api/ask:onFinish` + `:onError`. Anthropic counts captured; Voyage counts still TODO (see pre-compact handoff). Per-user / per-org / per-org_id rollups indexed. +- **Vercel KV rate limiting:** `lib/ai/rate-limit-kv.ts` — atomic INCR + EXPIRE via REST API, per-user keying for authenticated chat. Graceful in-memory fallback when KV isn't configured. +- **Per-org `enable_ask` gate:** `Settings.ENABLE_ASK_ORG_IDS` + `MeResponse.canUseAsk` on the backend; `canUseAskFor(req)` gate at `/api/ask` returns 403 `feature_not_enabled` early when the user's orgs aren't allowlisted (admins always pass; empty allowlist = open). +- HIPAA-aware compliance posture documented at `apps/web/docs/operations/hipaa-technical-safeguards.md` (control-by-control mapping) + `apps/web/docs/compliance/posture.md` (externalized for IRB / CISO) + `apps/web/docs/operations/audit-log-policy.md` (what IS / NEVER logged). The legacy `apps/web/COMPLIANCE.md` carries a header pointing to these docs. +- Architecture Decision Records at `apps/web/docs/architecture/decisions/001-008` covering heart-on-Railway, shared lib/ndi/, ToolContext, HttpOnly+CSRF, branch-aware preview, pgvector RAG (now **HNSW** post Stream 4.10), Vercel KV, and SYSTEM_PROMPT decomposition. +- pgvector index swapped IVFFlat → HNSW (Stream 4.10 migration at `apps/web/lib/ai/db/migrations/2026-05-15-hnsw.sql`). Expected ~30-80ms → ~5-15ms per `semantic_search_datasets`. +- **Single source of truth for current state**: [`apps/web/docs/HANDOFF.md`](apps/web/docs/HANDOFF.md). See the top-of-file pointer. +- Security incident closed: 2026-05-13/14 leaked Voyage + Railway-Postgres credentials in a pre-compact doc, rotated + BFG-rewritten + force-pushed. Full timeline at `apps/web/docs/security/2026-05-14-leaked-credentials-resolved.md`. Rollback tag `gitleaks-pre-scrub-2026-05-15-rollback` retained until 2026-05-22 then deleted. + +**Remaining backend work (deferred with specs)** — see HANDOFF.md "What's left" section: +- S4.9 — port `aggregate-documents.ts` to FastAPI (ADR-001 Heart-on-Railway compliance). ~1 day. +- S5.3 — BehavioralCompare cross-table joins. **SHIPPED** on `feat/ndi-python-phase-a` (commit `7157bde`). +- S5.8 — `/tables/{class}` server-side pagination. ~1 day. ~95% egress saving. + +S4.9 and S5.8 still need live data access; deferred to a session that has it. + +**Rules of engagement for any agent working on this branch (also documented in [`apps/web/docs/HANDOFF.md`](apps/web/docs/HANDOFF.md) §"Sacred rules"):** + +| Repo | `main` | Draft branch | +|---|---|---| +| `ndi-cloud-app` | production (DO NOT push) | `feat/experimental-ask-chat` (this) | +| `ndi-data-browser-v2` | production (DO NOT push) | `feat/ndi-python-phase-a` | + +- Production frontend URL: `https://ndi-cloud.com` (untouched) +- Preview frontend URL: `https://ndi-cloud-app-web-git-feat-experiment-c5da7d-ndi-cloud-a83eb4e7.vercel.app` +- Production backend: `https://ndb-v2-production.up.railway.app` (env id `e0c00fb7-ac98-431f-acdb-f4988032160f`) +- Experimental backend: `https://ndb-v2-experimental.up.railway.app` (env id `90101f6e-042b-44d6-8c8d-ec18d43b341b`) +- Test creds for Playwright smokes (workspace + chat): `audri+test@walthamdatascience.com / remhuz-ruwfy4-jiGcen` — Playwright form-fill ONLY, never write to disk, never echo in chat output. + +Reference plans (read in this order if picking up the branch cold): + +- **`apps/web/docs/HANDOFF.md`** — single source of truth for current state (start here). +- ADRs: `apps/web/docs/architecture/decisions/001-010-*` — architectural decisions, latest being ADR-010 (GitHub Template workflow). +- Operational reference (read when their topic comes up): + - `apps/web/docs/operations/hipaa-technical-safeguards.md` + - `apps/web/docs/operations/audit-log-policy.md` + - `apps/web/docs/operations/tenant-aware-tools-audit.md` + - `apps/web/docs/operations/three-surfaces.md` + - `apps/web/docs/operations/adding-a-workspace-panel.md` + - `apps/web/docs/operations/tutorial-parity-smoke.md` + - `apps/web/docs/operations/workspace-tutorial.md` + - `apps/web/docs/operations/vendor-dependencies.md` + - `apps/web/docs/operations/disaster-recovery.md` + - `apps/web/docs/operations/ndi-python-api-audit.md` — SDK surface audit driving lib/files.py shape + - `apps/web/docs/operations/ndi-matlab-api-audit.md` — same for MATLAB + - `apps/web/docs/operations/code-export-coverage-matrix.md` — (panel, tool) snippet coverage +- Compliance posture (externalized): `apps/web/docs/compliance/posture.md` +- Architectural rationale (legacy): `ndi-data-browser-v2/docs/plans/cross-repo-unification-2026-04-24.md` - v2 audit preserved: `ndi-data-browser-v2/docs/reviews/Audit_2026-04-23.md` -- Frontend polish audit: `apps/web/docs/reviews/Audit_2026-04-27_frontend_polish.md` (23/24 SHIPPED, 1 deferred-by-design as of `main` post-PR-#100) +- Frontend polish audit: `apps/web/docs/reviews/Audit_2026-04-27_frontend_polish.md` (23/24 SHIPPED) + +Older dated docs (`*-handoff*.md`, `*-pre-compact-*.md`, dated `*.md` +under `specs/` and `reviews/`) carry a SUPERSEDED header pointing back +to HANDOFF.md and are kept for archaeology only. ## Stack - **Framework:** Next.js 16.2.4 App Router (Turbopack), React 19 +- **AI:** **AI SDK v6** (`ai@6 @ai-sdk/anthropic@3 @ai-sdk/react@3`); upgrade landed 2026-05-15. Streaming via `streamText` with `await convertToModelMessages()`. Tool handlers in `lib/ndi/tools/*` (one per file, ~14 total). Anthropic Sonnet 4.x as the chat model. Voyage `voyage-4-large` for embeddings + `voyage rerank-2.5` for hybrid retrieval. RAG store on pgvector (Railway Postgres, HNSW index). - **Styling:** Tailwind v4 with `@theme` design tokens. NO SCSS Modules. NO MUI in `components/app/` (eslint enforced; MUI permitted only in `components/marketing/` for ``/`` where the a11y lift is real). -- **Data:** TanStack Query 5 (with PersistQueryClient layered on top in Phase 3a). Native `fetch()` via `apiFetch()`. No axios. -- **Tests:** Vitest + Testing Library (jsdom) for unit; Playwright for E2E. +- **Data:** TanStack Query 5 (with PersistQueryClient layered on top in Phase 3a). Native `fetch()` via `apiFetch()`. No axios. **Postgres (Railway)** via `pg` pool at `apps/web/lib/ai/db/pool.ts` — also serves `chunks` (RAG), `dataset_health_violations`, and `chat_usage_events`. +- **Rate limit:** Per-user via Vercel KV (`lib/ai/rate-limit-kv.ts`) with graceful in-memory fallback when KV isn't configured. +- **Cost tracking:** `lib/usage/{rate-card,log}.ts` writes one `chat_usage_events` row per /api/ask invocation. Anthropic rates pinned at module-level; Voyage rates likewise. Server-side computation of `total_cost_cents`. +- **Tests:** Vitest + Testing Library (jsdom) for unit (cloud-app, 1,612 tests); Playwright for E2E. pytest for ndb-v2 (893 tests). - **Bundle gate:** `scripts/check-bundle-size.mjs` — marketing 80 KB gz, app 200 KB gz. Ratchets DOWN over time, never up. - **Package manager:** pnpm 10.22 via Corepack. +- **pnpm-lock.yaml gotcha:** the lockfile lives at the repo root (NOT inside `apps/web/`). After ANY `pnpm add` / `pnpm remove`, you MUST `git add pnpm-lock.yaml` from the repo root (or `git add -A` from the repo root, NOT from `apps/web/`). Phase G + Phase H both shipped commits where the lockfile silently dropped because `git add -A apps/web` scoped to the wrong dir, and Vercel CI failed with `ERR_PNPM_OUTDATED_LOCKFILE`. Fixed in commit `61562ff` with a documented process note. ## Route groups -- `app/(marketing)/*` → `ndi-cloud.com` content (RSC-first, ISR where possible) -- `app/(app)/*` → former `app.ndi-cloud.com` content (mostly client; catalog is RSC + ISR) +- `app/(marketing)/*` → `ndi-cloud.com` content (RSC-first, ISR where possible). Includes `/(marketing)/ask` (anonymous-capable chat during transition). +- `app/(app)/*` → former `app.ndi-cloud.com` content (mostly client; catalog is RSC + ISR). Includes: + - `/my/workspace/[id]/...` — auth-gated workspace with 7 panels (Stream 6+) + - `/my/ask` — auth-gated chat route (Stream 3.1, 2026-05-15) + - `/admin/data-health` — admin Dataset Health dashboard (Stream 6.9) +- `app/api/cron/` — Vercel-scheduled crons (`warm-cache` every 5min; `dataset-health` 07:23 UTC daily). +- `app/api/admin/` — admin-authz read routes (currently `data-health`). +- `app/api/ask/` — anonymous-capable chat endpoint (gated by `askEnabled()` + `canUseAskFor(req)` for per-org access). +- `app/api/datasets/[id]//` — workspace wrapper routes for psth, spike-summary, tabular-query, treatment-timeline (auth-forwarding via `toolContextFromRequest`). `app.ndi-cloud.com` becomes a 301-to-apex redirect at Phase 7 cutover. Until then, both old domains keep serving production traffic from their respective old projects — this repo only deploys to Vercel preview URLs during Phases 1-6. @@ -61,6 +214,8 @@ Reference plans: HttpOnly `session` cookie set by FastAPI, scoped to `Domain=.ndi-cloud.com` (Phase 4). CSRF via double-submit `XSRF-TOKEN` cookie + echoed `X-XSRF-TOKEN` header. **No localStorage tokens** — Phase 2b rewrites the marketing-side auth flow that previously used localStorage Bearer tokens. +**Per-org `enable_ask` gate (Stream 3.4):** the backend's `MeResponse.canUseAsk` is true iff `is_admin` OR the user has at least one org in the FastAPI `Settings.ENABLE_ASK_ORG_IDS` allowlist (empty allowlist = open). The cloud-app's `/api/ask` route gates on this via `canUseAskFor(req)` and returns 403 `feature_not_enabled` early. The `/my/ask` page renders a "contact ops" notice when `canUseAsk === false`. + ## Author rule (non-negotiable) Every commit MUST be authored as `audriB `. Use `--author=` explicitly: @@ -116,3 +271,25 @@ Phase 7 shipped 2026-05-11. The remaining post-cutover work is non-traffic-movin ## Rollback (read this before any production-affecting change) The full rollback procedure lives outside this repo at `~/Documents/ndi-projects/cutover-keys.md` (owner-only `chmod 600`). It contains the pre-rotation `SESSION_ENCRYPTION_KEY` for restoring decryptable sessions if a Vercel domain detach is ever needed. Move both keys to a vault after the 30-day burn-in. + +Operational disaster-recovery runbooks (per failure mode, with RTO + RPO targets) live at `apps/web/docs/operations/disaster-recovery.md`. Five secret-rotation procedures (`SESSION_ENCRYPTION_KEY`, `CSRF_SIGNING_KEY`, `VOYAGE_API_KEY`, `ANTHROPIC_API_KEY`, `DATABASE_URL`) are documented there. + +## Postgres migrations + +Run order against the experimental Railway env (and later production). Idempotent — safe to re-run. + +```bash +# /ask RAG store (already applied) +psql "$DATABASE_URL" -f apps/web/lib/ai/db/schema.sql + +# Stream 4.10 — pgvector IVFFlat → HNSW +psql "$DATABASE_URL" -f apps/web/lib/ai/db/migrations/2026-05-15-hnsw.sql + +# Stream 6.8 — Dataset Health +psql "$DATABASE_URL" -f apps/web/lib/ai/db/migrations/2026-05-15-dataset-health.sql + +# Stream 3.2 — chat_usage_events +psql "$DATABASE_URL" -f apps/web/lib/ai/db/migrations/2026-05-15-chat-usage-events.sql +``` + +See `apps/web/lib/ai/db/migrations/README.md` for the operational guide. diff --git a/apps/web/.bundle-size-baseline.json b/apps/web/.bundle-size-baseline.json index 8ae70314..f8b73f9a 100644 --- a/apps/web/.bundle-size-baseline.json +++ b/apps/web/.bundle-size-baseline.json @@ -1,6 +1,6 @@ { "_comment": "Bundle-size ratchet baseline. Updated by `pnpm bundle-size --update` from the repo root after a successful build. Never edit by hand. Phase 6.7 A2 introduced the ratchet (replacing the hard 200 KB constant); the value below is the byte count from the last passing CI build, recomputed by check-bundle-size.mjs. Local builds may measure +/- ~hundred bytes due to gzip-encoder cross-platform variance — the script's RATCHET_SLACK_BYTES (1 KB) absorbs this.", - "_updated": "2026-04-28", - "_context": "Phase 6.7 Sequence 5 — A2 ratchet introduction", - "rootMainGzBytes": 172007 + "_updated": "2026-05-22", + "_context": "Audit 2026-05-20 — refresh after 4 weeks of additions (GitHub Template + AI SDK v6 + 9-panel workspace + KV rate limiter + cost tracking). +222 bytes net.", + "rootMainGzBytes": 172229 } diff --git a/apps/web/.env.example b/apps/web/.env.example index e7174f58..c3cfefdd 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -3,15 +3,83 @@ # # Phase 4 wires UPSTREAM_API_URL into next.config.ts rewrites. # Phase 3a wires INTERNAL_API_URL into RSC server-side prefetches. -# Phase 5 wires EDGE_CONFIG into lib/flags.ts. +# The /ask experimental chat reads ANTHROPIC_API_KEY, VOYAGE_API_KEY, +# DATABASE_URL, and NEXT_PUBLIC_ASK_ENABLED. +# The cron warm-cache route reads CRON_SECRET. -# FastAPI proxy base (Railway) — required for /api/* rewrite (Phase 4) +# ────────────────────────────────────────────────────────────────── +# Backend (FastAPI proxy on Railway) — required for /api/* rewrite +# ────────────────────────────────────────────────────────────────── + +# Public/edge rewrite target — Vercel proxies `/api/*` here. UPSTREAM_API_URL=https://ndb-v2-production.up.railway.app # Same as UPSTREAM_API_URL in production. Used by RSC server-side fetches # to bypass the Vercel rewrite layer (avoids double-hop). Phase 3a. INTERNAL_API_URL=https://ndb-v2-production.up.railway.app -# Vercel Edge Config connection string (Phase 5). -# Get from Vercel dashboard → Edge Config → Connection String. -# EDGE_CONFIG=https://edge-config.vercel.com/... +# ────────────────────────────────────────────────────────────────── +# Cron — /api/cron/warm-cache shared secret +# ────────────────────────────────────────────────────────────────── + +# Bearer secret that external cron callers must echo as +# `Authorization: Bearer ${CRON_SECRET}`. Vercel's own cron (set in +# vercel.json) sets `x-vercel-cron: 1` and bypasses this — so the +# variable can be unset for Vercel-managed cron only. +# CRON_SECRET= + +# ────────────────────────────────────────────────────────────────── +# /ask experimental chat (anonymous-public on feat/experimental-ask-chat) +# ────────────────────────────────────────────────────────────────── + +# Anthropic API key (Sonnet 4.x). When unset OR empty, /api/ask returns +# 503 and /ask renders a "coming soon" notice. Min length 20 chars. +# ANTHROPIC_API_KEY=sk-ant-api03-... + +# Public flag toggling the "Ask" link in the marketing header. Set +# to '1' to surface the tab; '0' or unset hides it. Decoupled from +# ANTHROPIC_API_KEY so the key can be deployed without the tab +# visible to general visitors. +# NEXT_PUBLIC_ASK_ENABLED=0 + +# Voyage AI key for query-time embedding + reranking (voyage-4-large + +# voyage rerank-2.5). Same key shape as vh-lab + shrek-lab chatbots. +# When unset, semantic_search_datasets returns an error and Claude +# falls back to structured catalog tools. Min length 10 chars. +# VOYAGE_API_KEY=pa-... + +# Postgres + pgvector connection string for the /ask RAG store. +# Each chatbot owns its own Railway-hosted pgvector instance. +# Required at runtime when semantic_search_datasets is exercised, and +# at build time when running `pnpm build-ask-index`. +# DATABASE_URL=postgresql://user:pass@host:port/dbname?sslmode=require + +# ────────────────────────────────────────────────────────────────── +# GitHub Template workflow (ADR-010) +# ────────────────────────────────────────────────────────────────── +# Powers the "Open in GitHub" + "Download as ZIP" buttons on every +# workspace panel + chat tool message. The buttons let users derive +# their own private repo from `Waltham-Data-Science/ndi-analysis-template` +# pre-populated with `current_analysis.py` matching the panel they +# were inspecting. See apps/web/docs/architecture/decisions/010-... +# +# GITHUB_CLIENT_ID + GITHUB_CLIENT_SECRET come from a GitHub OAuth App +# (Settings → Developer settings → OAuth Apps). Authorization callback +# URL must include `/api/github/oauth/callback` on every deploy. When +# either is unset, the "Open in GitHub" button renders disabled with +# a tooltip; the "Download as ZIP" button still works if GITHUB_APP_TOKEN +# is set. Min length 10 chars (GitHub IDs are ~20 chars). +# GITHUB_CLIENT_ID=Iv1.deadbeefdeadbeef +# GITHUB_CLIENT_SECRET= + +# Server-side PAT used to read the PRIVATE template repo for the +# "Download as ZIP" flow (no user OAuth). Scopes: `repo` (read). +# When unset, the /api/github/download-analysis-zip route returns +# 503 with a typed envelope. Min length 20 chars. +# GITHUB_APP_TOKEN=ghp_ + +# Public flag that the OpenInGitHubButton reads to decide whether to +# render enabled or disabled. Mirrors GITHUB_CLIENT_ID presence on the +# server. Decoupled so staging can set the secrets server-side while +# still hiding the button from end users. Set to '1' to enable. +# NEXT_PUBLIC_GITHUB_INTEGRATION_ENABLED=0 diff --git a/apps/web/.gitignore b/apps/web/.gitignore new file mode 100644 index 00000000..e985853e --- /dev/null +++ b/apps/web/.gitignore @@ -0,0 +1 @@ +.vercel diff --git a/apps/web/COMPLIANCE.md b/apps/web/COMPLIANCE.md index 10bbdee1..250832aa 100644 --- a/apps/web/COMPLIANCE.md +++ b/apps/web/COMPLIANCE.md @@ -1,4 +1,18 @@ -# Compliance posture — `ndi-cloud-app` (2026-04-26) +# Compliance posture — `ndi-cloud-app` (internal, 2026-04-26) + +> **2026-05-15 update — this document is now SUPPLEMENTARY.** +> The authoritative externally-distributable compliance posture is +> **`apps/web/docs/compliance/posture.md`** (Stream 2.6 deliverable). +> The control-by-control mapping of how each §164.312 requirement is +> realized in code lives in +> **`apps/web/docs/operations/hipaa-technical-safeguards.md`** +> (Stream 2.1 deliverable). +> +> This file is preserved for the data-residency / encryption / audit-trail +> reference tables which the externalized doc summarizes but does not +> reproduce in full. Internal contributors should use this file; external +> reviewers (IRB, CISO, prospective enterprise partners) should be sent +> the doc under `docs/compliance/`. This document records the data-handling, encryption, access-control, audit-trail, and regulatory-fit posture of the unified @@ -254,8 +268,15 @@ scratch. audit, O5 origin enforcement, O6 IDOR investigation. (`Waltham-Data-Science/ndi-data-browser-v2/docs/plans/cross-repo-unification-2026-04-24.md`) -## 8. Update history +## 8. External services + +| Service | Purpose | Data shared | Direction | +|---|---|---|---| +| **GitHub (OAuth + REST)** | "Open in GitHub" + "Download as ZIP" — ADR-010 | The user's own OAuth token (HttpOnly cookie, encrypted at rest with `GITHUB_TOKEN_ENCRYPTION_KEY`); the panel args + datasetName when the user clicks. No PHI; the dataset args are pointer references the user just saw in the workspace. | Outbound only; GitHub never reads cloud-app data. | + +## 9. Update history | Date | Change | Reason | |---|---|---| | 2026-04-26 | First draft. | Phase 6.7 Sequence 5 audit follow-up A10. | +| 2026-05-19 | Added §8 External services for GitHub OAuth + PAT. | ADR-010 — GitHub Template workflow. | diff --git a/apps/web/app/(app)/admin/data-health/data-health-client.tsx b/apps/web/app/(app)/admin/data-health/data-health-client.tsx new file mode 100644 index 00000000..ef6db03f --- /dev/null +++ b/apps/web/app/(app)/admin/data-health/data-health-client.tsx @@ -0,0 +1,285 @@ +'use client'; + +/** + * /admin/data-health client — table view over the + * `dataset_health_violations` snapshot. Grouped by severity: + * - critical (red) — must-fix data integrity issues + * - warning (amber) — likely ingest gaps; investigate + * - info (blue) — known-good asymmetries (e.g. C. elegans + * datasets with elements but no epochs) + * + * Fetches via TanStack Query (cookies forwarded automatically by + * apiFetch); the admin gate is server-side at + * `/api/admin/data-health/route.ts` which returns 403 for non- + * admin users. We surface that as an inline error rather than + * router-pushing to /login so an admin clicking around without an + * org switch sees the message and acts on it. + */ +import { AlertTriangle, Info, ShieldAlert } from 'lucide-react'; +import Link from 'next/link'; +import { useMemo } from 'react'; +import { useQuery } from '@tanstack/react-query'; + +import { ApiError, apiFetch } from '@/lib/api/client'; +import { Card, CardBody, CardHeader, CardTitle } from '@/components/ui/Card'; +import { Skeleton } from '@/components/ui/Skeleton'; + +interface ViolationRow { + datasetId: string; + datasetName: string | null; + invariantKey: string; + invariantLabel: string; + severity: 'critical' | 'warning' | 'info'; + message: string; + observation: Record; + snapshotAt: string; +} + +interface AdminResponse { + violations: ViolationRow[]; +} + +const SEVERITY_ORDER = ['critical', 'warning', 'info'] as const; + +export function DataHealthClient() { + const { data, isLoading, isError, error } = useQuery({ + queryKey: ['admin', 'data-health'], + queryFn: () => apiFetch('/api/admin/data-health'), + retry: false, + staleTime: 60_000, + }); + + const groups = useMemo(() => { + const out: Record = { + critical: [], + warning: [], + info: [], + }; + for (const v of data?.violations ?? []) { + const bucket = out[v.severity]; + if (bucket) bucket.push(v); + } + return out; + }, [data]); + + return ( +
+
+

+ Data health +

+

+ Latest Dataset Health invariant snapshot. The nightly cron at{' '} + /api/cron/dataset-health scans + every published dataset and writes violations here. Datasets + with no current violations don’t appear — the table + always reflects the latest per-dataset state. +

+
+ + {isLoading && ( +
+ + +
+ )} + + {isError && ( + + )} + + {!isLoading && !isError && data && ( + <> + v.datasetId)).size + } + /> + {SEVERITY_ORDER.map((severity) => { + const rows = groups[severity] ?? []; + if (rows.length === 0) return null; + return ( + + ); + })} + {(data.violations ?? []).length === 0 && ( + + +

+ All datasets healthy 🎉 +

+

+ The last cron run found no invariant violations across + the published catalog. +

+ + + )} + + )} +
+ ); +} + +function ErrorBanner({ err }: { err: unknown }) { + let title = 'Something went wrong loading data health.'; + let detail: string | null = null; + if (err instanceof ApiError) { + if (err.status === 403) { + title = 'Admin access required.'; + detail = + 'Sign in with an admin account or ask an admin to grant you the role.'; + } else { + title = err.message || title; + } + } else if (err instanceof Error) { + detail = err.message; + } + return ( +
+

{title}

+ {detail &&

{detail}

} +
+ ); +} + +interface SummaryStripProps { + critical: number; + warning: number; + info: number; + totalAffected: number; +} + +function SummaryStrip({ critical, warning, info, totalAffected }: SummaryStripProps) { + return ( +
+ + + + +
+ ); +} + +function StatChip({ + label, + value, + tint, + Icon, +}: { + label: string; + value: number; + tint: string; + Icon: typeof ShieldAlert; +}) { + return ( +
+
+ + {label} +
+
+ {value} +
+
+ ); +} + +interface SeverityGroupProps { + severity: 'critical' | 'warning' | 'info'; + rows: ViolationRow[]; +} + +function SeverityGroup({ severity, rows }: SeverityGroupProps) { + const label = + severity === 'critical' + ? 'Critical' + : severity === 'warning' + ? 'Warning' + : 'Info'; + return ( + + + + {label} · {rows.length} violation{rows.length === 1 ? '' : 's'} + + + + + + + + + + + + + {rows.map((r) => ( + + + + + + ))} + +
DatasetInvariantMessage
+ + {r.datasetName ?? r.datasetId} + +
+ {r.datasetId} +
+ 		+
+ {r.invariantLabel} +
+
+ {r.invariantKey} +
+ 		+ {r.message} +
+ + + ); +} diff --git a/apps/web/app/(app)/admin/data-health/page.tsx b/apps/web/app/(app)/admin/data-health/page.tsx new file mode 100644 index 00000000..d6e3fa2a --- /dev/null +++ b/apps/web/app/(app)/admin/data-health/page.tsx @@ -0,0 +1,29 @@ +import type { Metadata } from 'next'; + +import { DataHealthClient } from './data-health-client'; + +/** + * /admin/data-health — Dataset Health admin dashboard. + * + * Stream 6.9 (2026-05-15) deliverable. Reads the latest snapshot + * from `/api/admin/data-health` (which fronts the + * `dataset_health_violations` Postgres table populated nightly by + * the cron at `/api/cron/dataset-health`). + * + * The full invariant set fires here (not just the compact-safe + * subset that powers the catalog badge) — see + * `apps/web/lib/data-quality/invariants.ts` for the catalog vs. + * full split, ADR-009 (planned) for the rationale. + * + * Authz is enforced server-side at `/api/admin/data-health/route.ts` + * (returns 403 unless the session user is admin). The page itself + * renders to anyone; the admin gate is the data source. + */ +export const metadata: Metadata = { + title: 'Data health · admin', + robots: { index: false, follow: false }, +}; + +export default function DataHealthPage() { + return ; +} diff --git a/apps/web/app/(app)/datasets/[id]/documents/[docId]/document-detail-shell.tsx b/apps/web/app/(app)/datasets/[id]/documents/[docId]/document-detail-shell.tsx index 717fd287..7740c197 100644 --- a/apps/web/app/(app)/datasets/[id]/documents/[docId]/document-detail-shell.tsx +++ b/apps/web/app/(app)/datasets/[id]/documents/[docId]/document-detail-shell.tsx @@ -66,6 +66,24 @@ export function DocumentDetailShell({ const docClass = doc.data?.className; const eyebrowTail = docClass ?? (docId.length > 24 ? `${docId.slice(0, 24)}…` : docId); + // Smarter H1 fallback chain — many NDI doc classes (epoch, vmspikesummary, + // element_epoch, ontologyTableRow, treatment timeline) have no useful + // `name` field. Some return the literal "Document" placeholder, others + // return undefined. Before the fix both paths rendered as just + // "Document" in the H1 (visual-UX audit, a395 P0 #5, 2026-05-14). + // + // Treat the literal "Document" (any casing) as equivalent to no name — + // it carries no information beyond what the eyebrow already shows. + // The H1 then falls back to " " so each + // document has a distinguishable headline. + const shortDocId = + docId.length > 16 ? `${docId.slice(0, 8)}…${docId.slice(-4)}` : docId; + const isGenericPlaceholderName = + !docName || docName.trim().toLowerCase() === 'document'; + const h1Fallback = docClass + ? `${docClass} ${shortDocId}` + : `Document ${shortDocId}`; + const h1Text = isGenericPlaceholderName ? h1Fallback : docName; return ( <> @@ -85,7 +103,9 @@ export function DocumentDetailShell({ opacity: 0.05, }} /> -
+ {/* Match the mobile px ramp on the body section below: `px-4` + on phones, `px-7` from sm: upward. */} +
- {docName ?? 'Document'} + {h1Text} )} @@ -141,7 +161,10 @@ export function DocumentDetailShell({ visual). Side-by-side keeps both above the fold on most desktops + makes the page feel materially richer. */} -
+ {/* `px-7` desktop; `px-4` below sm: matches the dataset chrome + gate's mobile padding so the document-detail body uses the + same content width as the surrounding tab UI. */} +
` (DocumentExplorer.tsx + // ~198) switches to side-by-side at `md:` (768px), not `lg:` + // (1024px) — the skeleton must match so the layout doesn't reflow + // when the data lands on tablet widths.
{/* Sidebar: class filter list. */} -
{/* ── Body ─────────────────────────────────────────────────────── */} -
-
- setStatusFilter('all')} - count={counts.all} - > - All - - setStatusFilter('published')} - count={counts.published} + {/* `px-7` is the desktop chrome value; `px-4` below sm: matches + the hero band's mobile padding ramp so the list flush-aligns + with the stat strip above on narrow viewports. */} +
+ {/* Top-of-section tab strip — switches the dataset source + between the user's own datasets and the public NDI catalog. + Both feed the same card/table render below; the only thing + that changes is the data query the chips/cards bind to. */} +
+ setActiveTab('mine')} > - Published - - setStatusFilter('draft')} - count={counts.draft} + Your datasets + {myDatasetsQuery.data && ( + + {formatNumber(myDatasetsQuery.data.datasets.length)} + + )} + + setActiveTab('public')} > - Draft / in-review - + Public NDI catalog + {publicDatasetsQuery.data && ( + + {formatNumber( + publicDatasetsQuery.data.totalNumber ?? + publicDatasetsQuery.data.datasets.length, + )} + + )} +
+ {/* Status filter chips only meaningful for "Your datasets" — + public catalog entries are all published by definition, so + the All/Published/Draft toggle would be a no-op there. */} + {activeTab === 'mine' && ( +
+ setStatusFilter('all')} + count={counts.all} + > + All + + setStatusFilter('published')} + count={counts.published} + > + Published + + setStatusFilter('draft')} + count={counts.draft} + > + Draft / in-review + +
+ )} + {datasetsQuery.isError && (

@@ -299,17 +380,54 @@ export function MyDatasetsClient() { (viewMode === 'grid' ? (

{visible.map((d) => ( - + ))}
) : ( - + ))}
); } +/* ─── Tab buttons (top of body) ──────────────────────────────────── */ + +function TabButton({ + active, + onClick, + children, +}: { + active: boolean; + onClick: () => void; + children: React.ReactNode; +}) { + return ( + + ); +} + /* ─── HeroStat (glassmorphic stat card) ──────────────────────────── */ function HeroStat({ diff --git a/apps/web/app/(app)/my/workspace/[id]/layout.tsx b/apps/web/app/(app)/my/workspace/[id]/layout.tsx new file mode 100644 index 00000000..c8293152 --- /dev/null +++ b/apps/web/app/(app)/my/workspace/[id]/layout.tsx @@ -0,0 +1,90 @@ +/** + * Workspace layout — chrome for `/my/workspace/[id]` (Phase F redesign). + * + * Pre-redesign this layout wrapped a 5-tab IA (Overview / Structure / + * Subjects / Sessions / Analyses). The Phase F redesign collapses + * the tabs into a single canvas (rendered by `page.tsx`), so this + * layout is now thinner — just the hero, the auth gate, and the + * AskPanel + keyboard shortcuts. + * + * Why the auth gate wraps only `children` (not hero / AskPanel): + * - The hero pulls public dataset metadata (`safeFetchDataset`), + * the same data `/datasets/[id]` already serves anonymously. + * Showing it briefly to an unauthenticated visitor is fine. + * - The AskPanel is also workspace-level chrome that survives auth + * resolve — its empty state handles the not-yet-signed-in case. + * - The canvas (children) holds the workspace tables + analyses, + * which need auth; the gate sits over those alone. + * + * Why `
` around the gate-wrapped children: the canvas + * holds 6 panels each with its own form/mutation state. When the + * user navigates from `/my/workspace/A` → `/my/workspace/B` we want + * a full subtree remount so stale mutation state from A doesn't + * leak under B's hero. Keying the wrapper by `id` forces it. + */ +import { Suspense } from 'react'; + +import { AskKeyboardShortcuts } from '@/components/ai/AskKeyboardShortcuts'; +import { AskPanel } from '@/components/ai/AskPanel'; +import { AskPanelTrigger } from '@/components/ai/AskPanelTrigger'; +import { WorkspaceAuthGate } from '@/components/workspace/WorkspaceAuthGate'; +import { + WorkspaceShell, + WorkspaceShellSkeleton, +} from '@/components/workspace/WorkspaceShell'; +import { safeFetchDataset } from '@/lib/api/datasets-server'; +import { cleanDatasetName } from '@/lib/format'; + +interface LayoutProps { + children: React.ReactNode; + params: Promise<{ id: string }>; +} + +export default async function WorkspaceLayout({ + children, + params, +}: LayoutProps) { + const { id } = await params; + + // Pre-fetch dataset name so AskPanel's context line ("Asking + // about: ") renders correctly on first paint. Same fetch + // is cached for WorkspaceShell's render below (same RSC request). + const datasetForContext = await safeFetchDataset(id).catch(() => null); + const datasetName = datasetForContext + ? cleanDatasetName(datasetForContext.name) + : undefined; + + return ( + <> + }> + + +
+ {children} +
+ + {/* + AskPanel + Trigger + KeyboardShortcuts — workspace-level chat + affordance. All three call `useSearchParams()` via + `useAskPanelState`, so they MUST live inside a `` + per the App Router's CSR-bailout rule for that hook. The + single shared Suspense keeps them out of any potential + bailout that would force the whole layout into client-side + rendering. + + Phase F (W7 fix): AskPanel's `context` now carries selection + bar state in addition to dataset id/name — see the AskShell + refactor for how the chat request body picks this up. + */} + + + + + + + ); +} diff --git a/apps/web/app/(app)/my/workspace/[id]/page.tsx b/apps/web/app/(app)/my/workspace/[id]/page.tsx new file mode 100644 index 00000000..d019aa2b --- /dev/null +++ b/apps/web/app/(app)/my/workspace/[id]/page.tsx @@ -0,0 +1,54 @@ +/** + * `/my/workspace/[id]` — the workspace canvas (Phase F redesign). + * + * Previously this was a server-side redirect to + * `/my/workspace/[id]/overview`. The Phase F redesign collapses the + * 5-tab IA into a single canvas, so the bare id route now renders + * the canvas directly. + * + * The page is a thin server component — all the interactivity is in + * `WorkspaceCanvasClient` which uses `useWorkspaceSelection`. We + * resolve the `params` Promise here so the client receives a plain + * id string and renders without server-side hooks. + * + * The hero + AskPanel + AskKeyboardShortcuts mount in `layout.tsx`, + * not here — they're shared chrome that should survive intra- + * workspace state changes. + */ +import { Suspense } from 'react'; + +import { WorkspaceCanvasClient } from '@/components/workspace/canvas/WorkspaceCanvasClient'; + +interface PageProps { + params: Promise<{ id: string }>; +} + +/** + * Suspense fallback for the canvas — picker rail + main area in a + * coarse 2-column shape. The canvas's own components carry finer + * skeletons for stats/provenance/picker rows, so this top-level + * fallback only renders for the moment between route resolve and + * the canvas client booting. + */ +function CanvasFallback() { + return ( +
+ +
+
+
+
+ ); +} + +export default async function WorkspacePage({ params }: PageProps) { + const { id } = await params; + + return ( + }> + + + ); +} diff --git a/apps/web/app/(marketing)/ask/page.tsx b/apps/web/app/(marketing)/ask/page.tsx new file mode 100644 index 00000000..5b4350f4 --- /dev/null +++ b/apps/web/app/(marketing)/ask/page.tsx @@ -0,0 +1,25 @@ +/** + * `/ask` — RETIRED (2026-05-16, Phase D workspace redesign). + * + * Ask is now a workspace-only affordance, accessible via the drawer + * trigger inside `/my/workspace/[id]/*`. The public anonymous chat + * surface that used to live at this URL is retired as part of the + * Phase D migration — Ask is no longer a public marketing-side + * surface (per the design doc's locked decision, with a dedicated + * marketing page slated to appear within the Data Browser product + * page once that product launches publicly). + * + * Anyone arriving at `/ask` (bookmarks, external links) is + * server-redirected to `/create-account?next=/my` so: + * - Authenticated visitors land in their dataset list after the + * auth pass-through. + * - New visitors are prompted to create an account before + * accessing the workspace chat. + * + * `redirect()` is a server-side redirect; no client flash. + */ +import { redirect } from 'next/navigation'; + +export default function RetiredAskPage(): never { + redirect('/create-account?next=/my'); +} diff --git a/apps/web/app/(marketing)/reset-password/reset-password-form.tsx b/apps/web/app/(marketing)/reset-password/reset-password-form.tsx index 52a385b5..8efbac13 100644 --- a/apps/web/app/(marketing)/reset-password/reset-password-form.tsx +++ b/apps/web/app/(marketing)/reset-password/reset-password-form.tsx @@ -2,13 +2,14 @@ import Link from 'next/link'; import { useRouter } from 'next/navigation'; -import { useState, type FormEvent } from 'react'; +import { useEffect, useState, type FormEvent } from 'react'; import { ApiError } from '@/lib/api/client'; import { changePassword } from '@/lib/api/auth'; import { AuthCard } from '@/components/marketing/AuthCard'; import { Field, FormError } from '@/components/marketing/AuthForm'; import { MarketingButton } from '@/components/marketing/Button'; +import { useSession } from '@/lib/auth/use-session'; const MIN_PASSWORD = 12; @@ -19,9 +20,21 @@ const MIN_PASSWORD = 12; * which uses an emailed code). This page requires the current * password as proof of session — protects against an attacker with a * stolen XSRF cookie but no password from rotating creds. + * + * # Anonymous-user posture + * + * Pre-2026-05-14, anonymous visitors saw the "Change password" form + * and were asked for their current password — confusing for anyone + * who arrived from the legacy `/resetPassword` camelCase alias or a + * search-result snippet (visual-UX audit #6, P0-1 from a63c agent). + * Now anonymous visitors are redirected to /login with returnTo set, + * and the form additionally renders a "Forgot your password?" link + * to /forgot-password so authenticated users who can't remember + * their current password have a clear escape hatch. */ export function ResetPasswordForm() { const router = useRouter(); + const { user, isLoading } = useSession(); const [currentPassword, setCurrentPassword] = useState(''); const [newPassword, setNewPassword] = useState(''); const [error, setError] = useState(null); @@ -29,6 +42,23 @@ export function ResetPasswordForm() { const [submitting, setSubmitting] = useState(false); const [success, setSuccess] = useState(false); + // Auth gate: anonymous users can't change a password they don't + // know — they need to recover via email instead. Follows the same + // pattern as `my-account-client.tsx`'s redirect-to-login. + useEffect(() => { + if (!isLoading && !user) { + router.replace('/login?returnTo=/reset-password'); + } + }, [isLoading, user, router]); + + if (isLoading || !user) { + return ( +
+

Loading…

+
+ ); + } + async function handleSubmit(e: FormEvent) { e.preventDefault(); setError(null); @@ -94,9 +124,18 @@ export function ResetPasswordForm() { heading="Change your password" description="Enter your current password, then choose a new one." footer={ - - Back to account - +
+ + Back to account + + + Forgot your current password?{' '} + + Reset it via email + + . + +
} >
diff --git a/apps/web/app/api/admin/data-health/route.ts b/apps/web/app/api/admin/data-health/route.ts new file mode 100644 index 00000000..6da0463d --- /dev/null +++ b/apps/web/app/api/admin/data-health/route.ts @@ -0,0 +1,71 @@ +/** + * GET /api/admin/data-health — read the latest Dataset Health snapshot. + * + * Stream 6.9 (2026-05-15). Returns every violation from the latest + * cron snapshot, ordered critical → warning → info. The + * `/admin/data-health` page consumes this. + * + * Authz: requires an authenticated admin session (the FastAPI proxy's + * existing session-cookie check + `is_admin` flag). The wrapper + * forwards the user's `Cookie` to FastAPI's `/api/auth/me` for the + * admin verification — same shape as other admin-only routes in this + * codebase. + */ +import { NextResponse, type NextRequest } from 'next/server'; + +import { logEvent } from '@/lib/ndi/tools/shared'; +import { readAllLatestViolations } from '@/lib/data-quality/persistence'; +import { env } from '@/lib/env'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +function baseUrl(): string | null { + if (env.VERCEL_GIT_COMMIT_REF === 'feat/experimental-ask-chat') { + return 'https://ndb-v2-experimental.up.railway.app'; + } + const u = env.INTERNAL_API_URL; + return typeof u === 'string' && u.length > 0 ? u : null; +} + +interface AuthMe { + user?: { isAdmin?: boolean }; + isAdmin?: boolean; +} + +async function isAdmin(req: NextRequest): Promise { + const base = baseUrl(); + if (!base) return false; + const cookie = req.headers.get('cookie'); + if (!cookie) return false; + try { + const res = await fetch(`${base}/api/auth/me`, { + headers: { Cookie: cookie, Accept: 'application/json' }, + cache: 'no-store', + }); + if (!res.ok) return false; + const body = (await res.json()) as AuthMe; + return Boolean(body.user?.isAdmin ?? body.isAdmin); + } catch { + return false; + } +} + +export async function GET(req: NextRequest) { + if (!(await isAdmin(req))) { + return NextResponse.json({ error: 'forbidden' }, { status: 403 }); + } + try { + const rows = await readAllLatestViolations(); + logEvent('dataset_health.admin.read', { row_count: rows.length }); + return NextResponse.json({ violations: rows }); + } catch (err) { + logEvent('dataset_health.admin.read_error', { + error: err instanceof Error ? err.message : 'unknown', + }); + return NextResponse.json( + { error: 'persistence_error' }, + { status: 503 }, + ); + } +} diff --git a/apps/web/app/api/ask/route.ts b/apps/web/app/api/ask/route.ts new file mode 100644 index 00000000..da3828f1 --- /dev/null +++ b/apps/web/app/api/ask/route.ts @@ -0,0 +1,705 @@ +/** + * POST /api/ask — experimental chat endpoint. + * + * Pipeline: + * 1. Feature-flag check (ANTHROPIC_API_KEY) → 503 if off. + * 2. Per-IP rate-limit → 429 if exceeded. + * 3. Body parse + minimal shape check → 400 if malformed. + * 4. streamText with bound tools → SSE stream back to client. + * + * Runtime: Node (not edge). Originally edge-runtime for streaming + * TTFB, but the RAG layer imports a multi-MB dataset-index.json + * (~500 datasets × 1024-d float32 embeddings + text + metadata). + * Bundling that into the edge function would push us against + * Vercel's 4 MB compressed-edge-function limit. Node serverless + * has a 250 MB limit and ~200-500ms cold start — fine for the + * demo cadence. Streaming still works the same way through the AI + * SDK; only the runtime label changes. + * + * Anonymous-only. No CSRF check (no cookies, no auth, public-data + * only). Origin enforcement at the Vercel middleware still applies. + */ +import { + convertToModelMessages, + stepCountIs, + streamText, + type ModelMessage, + type UIMessage, +} from 'ai'; + +import { chatModel, CLAUDE_MODEL_ID } from '@/lib/ai/anthropic-client'; +import { askEnabled } from '@/lib/ai/feature-flag'; +import { checkRateLimitKv } from '@/lib/ai/rate-limit-kv'; +import { SYSTEM_PROMPT } from '@/lib/ai/system-prompt'; +import { makeTools } from '@/lib/ai/chat-tools'; +import { env } from '@/lib/env'; +import { + authHeadersFromRequest, + logEvent, + type ToolContext, +} from '@/lib/ndi/tools/shared'; +import { logUsage } from '@/lib/usage/log'; +import type { ProviderUsage } from '@/lib/usage/rate-card'; + +// Audit 2026-05-20 P1 — single source of truth for the model id we +// report on each usage event. Re-exported from anthropic-client so a +// model bump in one place (the bound model handle) updates the cost +// telemetry column in lockstep. Pre-fix this was a placeholder +// string ('claude-sonnet-4.x') that never matched any real model id. +const ASK_MODEL_ID = CLAUDE_MODEL_ID; + +// Audit 2026-05-20 P1 — message-history size cap. Clients submit a +// `messages[]` array via DefaultChatTransport.body; without a cap a +// single crafted request can exceed Anthropic's 200K context mid- +// stream (worst-case ~$15+ on tokens). Counted by parts text length +// across all messages; tool-call results are excluded (they're not +// what the user authored). +const MAX_INBOUND_MESSAGES = 64; +const MAX_INBOUND_MESSAGE_CHARS = 60_000; + +function zeroProviderUsage(): ProviderUsage { + return { + anthropicInputTokens: 0, + anthropicOutputTokens: 0, + anthropicCacheReadTokens: 0, + anthropicCacheCreateTokens: 0, + voyageEmbedTokens: 0, + voyageRerankUnits: 0, + }; +} + +export const runtime = 'nodejs'; +// Allow up to 180s. Trajectory of bumps: +// 60s — initial cap; covered 4 tool roundtrips at ~8s each + compose. +// 180s — current; exploratory dataset overview prompts ("how many +// subjects, what classes, figure coverage…") chain 5-7 tools +// and at 60s the stream was being cut off mid-compose with +// no assistant summary text emitted (caught live during +// 2026-05-14 tutorial-parity smoke). 180s gives the model +// comfortable headroom; 99th-percentile latency on healthy +// chains is still ~25-40s so this only bites pathologically +// long traces. Vercel Pro tier allows up to 300s; 180s +// leaves margin to grow. +export const maxDuration = 180; + +/** + * Stream 3.4 (2026-05-15) — per-org access verdict for `/api/ask`. + * + * Returns one of: + * - `{ verdict: 'anonymous' }` — no session cookie. + * - `{ verdict: 'allowed', userId, orgId? }` — session ok + canUseAsk=true. + * - `{ verdict: 'forbidden', userId, orgId? }` — session ok + canUseAsk=false. + * - `{ verdict: 'unavailable' }` — upstream errored AND we have a + * cookie; can't decide → 503. + * + * Audit 2026-05-20 P0 #5: fail-CLOSED on non-401 upstream errors + * when the caller has a session cookie. Pre-fix, any 5xx (Railway + * outage etc.) returned 'allowed' for every cookie-bearing request, + * silently neutralizing the per-org ENABLE_ASK_ORG_IDS allowlist + * during outages. Anonymous-path callers (no cookie) still admit — + * the route is anonymous-capable by design and shouldn't be coupled + * to backend health for that surface. + */ +interface AskVerdict { + verdict: 'anonymous' | 'allowed' | 'forbidden' | 'unavailable'; + userId: string; + organizationId: string | null; +} + +async function canUseAskFor(req: Request): Promise { + const cookie = req.headers.get('cookie'); + if (!cookie) { + return { verdict: 'anonymous', userId: 'anonymous', organizationId: null }; + } + // Resolve the FastAPI base the same way the chat tools do — branch- + // aware so the experimental preview hits the experimental Railway env. + const upstream = + env.VERCEL_GIT_COMMIT_REF === 'feat/experimental-ask-chat' + ? 'https://ndb-v2-experimental.up.railway.app' + : env.INTERNAL_API_URL; + if (!upstream) { + return { verdict: 'anonymous', userId: 'anonymous', organizationId: null }; + } + try { + const res = await fetch(`${upstream}/api/auth/me`, { + headers: { Cookie: cookie, Accept: 'application/json' }, + cache: 'no-store', + }); + if (res.status === 401) { + return { verdict: 'anonymous', userId: 'anonymous', organizationId: null }; + } + if (!res.ok) { + // Audit 2026-05-20 P0 #5 — fail closed. The caller IS carrying + // a session cookie (we'd have returned anonymous otherwise), + // but FastAPI couldn't confirm canUseAsk. Pre-fix this returned + // 'allowed' which bypassed the org allowlist during Railway + // outages. 'unavailable' surfaces upstream to 503 so the gate + // stays honest. + return { verdict: 'unavailable', userId: 'anonymous', organizationId: null }; + } + const body = (await res.json()) as { + userId?: string; + canUseAsk?: boolean; + organizationIds?: string[]; + }; + const userId = + typeof body.userId === 'string' && body.userId + ? body.userId + : 'anonymous'; + const organizationId = + Array.isArray(body.organizationIds) && body.organizationIds.length > 0 + ? body.organizationIds[0]! + : null; + return { + verdict: body.canUseAsk === false ? 'forbidden' : 'allowed', + userId, + organizationId, + }; + } catch { + // Network/timeout/parse error with a cookie present — same + // posture as a 5xx above. Fail closed. + return { verdict: 'unavailable', userId: 'anonymous', organizationId: null }; + } +} + +/** + * Stream 3.2 — generate a stable request id for cross-boundary + * tracing. Same shape as the FastAPI middleware's regex + * (`[A-Za-z0-9_.-]{8,128}`); 16 hex chars is enough entropy at our + * request volume. + */ +function freshRequestId(): string { + if (typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function') { + return crypto.randomUUID().replace(/-/g, '').slice(0, 16); + } + let id = ''; + for (let i = 0; i < 16; i++) { + id += Math.floor(Math.random() * 16).toString(16); + } + return id; +} + +function clientIp(req: Request): string { + // Audit 2026-05-20 P1 — read RIGHTMOST x-forwarded-for entry as the + // canonical client IP. On Vercel, the platform sets the leftmost + // (or all) entries, but a request that comes through an additional + // upstream proxy (Cloudflare, etc.) carries an attacker-set IP in + // the leftmost slot. Vercel's edge appends the real client to the + // RIGHT of any upstream value. Reading rightmost is the standard + // recommendation for proxy chains where the last proxy is trusted. + const fwd = req.headers.get('x-forwarded-for'); + if (fwd) { + const parts = fwd.split(',').map((s) => s.trim()).filter((s) => s.length > 0); + if (parts.length > 0) return parts[parts.length - 1]!; + } + const real = req.headers.get('x-real-ip'); + if (real) return real.trim(); + return 'unknown'; +} + +export async function POST(req: Request): Promise { + // 1. Feature flag. + if (!askEnabled()) { + logEvent('ask.feature_disabled'); + return Response.json({ error: 'chat_disabled' }, { status: 503 }); + } + + // 1b. Stream 3.4 (2026-05-15) — per-org access gate. The route is + // STILL ANONYMOUS-CAPABLE during the experimental phase: requests + // without a session cookie skip the gate (the chat is open to + // anyone today). Once Stream 3.1 moves /ask under /my/ask the + // route becomes auth-required; this gate then enforces the + // FastAPI-side ENABLE_ASK_ORG_IDS allowlist (admins always pass; + // empty allowlist means "every authenticated user"). + const askVerdict = await canUseAskFor(req); + if (askVerdict.verdict === 'forbidden') { + logEvent('ask.feature_not_enabled_for_org', { userId: askVerdict.userId }); + return Response.json( + { error: 'feature_not_enabled' }, + { status: 403 }, + ); + } + if (askVerdict.verdict === 'unavailable') { + // Audit 2026-05-20 P0 #5 — fail-closed on 5xx from FastAPI/me when + // the caller has a session cookie. The org gate can't decide; we + // refuse rather than admit. Anonymous traffic (no cookie) takes + // the 'anonymous' branch above and is unaffected. + logEvent('ask.gate_unavailable'); + return Response.json( + { error: 'service_unavailable', message: 'Ask is temporarily unavailable. Try again in a minute.' }, + { status: 503, headers: { 'Retry-After': '30' } }, + ); + } + // Stream 3.2 — userId/organizationId reused by the usage event + // emitted from streamText's onFinish/onError below. requestId + // correlates with the X-Request-Id propagated through + // toolContextFromRequest into FastAPI logs. + const userId = askVerdict.userId; + const organizationId = askVerdict.organizationId; + const requestId = freshRequestId(); + const askStartedAtMs = Date.now(); + + // 2. Rate limit (before any expensive parsing). + // Stream 3.3 (2026-05-15): swapped the per-IP in-memory limiter + // for a per-USER KV-backed limiter (with in-memory fallback when + // KV isn't configured — local dev / preview). Authenticated chat + // keys on userId so multi-instance Vercel deploys honor the cap + // across the whole fleet. Anonymous chat still keys on IP. + const ip = clientIp(req); + const subject = userId !== 'anonymous' ? `user:${userId}` : `ip:${ip}`; + const rl = await checkRateLimitKv(subject); + if (!rl.ok) { + logEvent('ask.rate_limited', { + subject, + bucket: rl.bucket, + retryAfterSeconds: rl.retryAfterSeconds, + }); + return Response.json( + { + error: 'rate_limited', + bucket: rl.bucket, + retryAfterSeconds: rl.retryAfterSeconds, + }, + { status: 429, headers: { 'Retry-After': String(rl.retryAfterSeconds) } }, + ); + } + + // 3. Body parse + shape check. + let body: unknown; + try { + body = await req.json(); + } catch { + logEvent('ask.invalid_body', { reason: 'invalid_json' }); + return Response.json({ error: 'invalid_json' }, { status: 400 }); + } + + const messages = extractMessages(body); + if (!messages) { + logEvent('ask.invalid_body', { reason: 'shape_mismatch' }); + return Response.json({ error: 'invalid_body' }, { status: 400 }); + } + + // Audit 2026-05-20 P1 — message-history size cap. Refuse runaway + // payloads BEFORE we hand them to convertToModelMessages / Anthropic. + if (messages.length > MAX_INBOUND_MESSAGES) { + logEvent('ask.invalid_body', { + reason: 'messages_too_many', + count: messages.length, + cap: MAX_INBOUND_MESSAGES, + }); + return Response.json( + { error: 'invalid_body', message: 'Conversation is too long. Start a new chat.' }, + { status: 413 }, + ); + } + const totalTextChars = totalUserTextChars(messages); + if (totalTextChars > MAX_INBOUND_MESSAGE_CHARS) { + logEvent('ask.invalid_body', { + reason: 'messages_too_large', + chars: totalTextChars, + cap: MAX_INBOUND_MESSAGE_CHARS, + }); + return Response.json( + { error: 'invalid_body', message: 'Conversation is too long. Start a new chat.' }, + { status: 413 }, + ); + } + + // Phase F (W7 audit fix) — pull optional workspace context out of + // the request body. `AskShell` passes this via + // `DefaultChatTransport.body`. Fields are independently optional; + // a chat from outside a workspace will carry none of them. + const workspaceContext = extractWorkspaceContext(body); + + // Request observability — size-only, never message content. + const lastUserMessage = lastUserText(messages); + logEvent('ask.request.start', { + ip, + messageCount: messages.length, + mostRecentUserMessage_length: lastUserMessage.length, + hasWorkspaceContext: workspaceContext !== null, + workspaceContextKeys: workspaceContext + ? Object.keys(workspaceContext).length + : 0, + }); + + // 4. Stream. + // + // # Anthropic prompt caching (added 2026-05-14) + // + // The SYSTEM_PROMPT is ~10K tokens of stable instructions (tool + // usage hints, citation rules, dataset disambiguation). Pre-cache, + // every tool roundtrip paid the full input cost again — and a + // multi-tool turn can roundtrip 4-7 times. At Sonnet 4.5 pricing + // ($3/MTok input), that's ~30¢ per turn just on the system prompt. + // With `cacheControl: { type: 'ephemeral' }` on the system message, + // Anthropic caches the prompt for 5 minutes after first write and + // bills cache reads at 10% of the input rate (~$0.30/MTok). Within + // a conversation, the second turn onward hits the cache → input + // cost on system drops to ~3¢ per turn (a ~10× reduction on the + // system slice of the budget). + // + // The cache breakpoint here goes on the system message ONLY — that + // captures the largest stable prefix without forcing us to manage + // breakpoints across the user's growing message history. Anthropic + // allows up to 4 breakpoints per request; if we wanted to also cache + // accumulated history we'd add one to the last assistant message. + // Future work — for now the single-breakpoint win is large enough. + // + // The `system` arg is replaced by a `system`-role message at the + // front of `messages` because that's where the AI SDK exposes + // per-message `providerOptions`. Functionally equivalent — the + // Anthropic-side API receives the system instruction the same way. + const systemMessage: ModelMessage = { + role: 'system', + content: SYSTEM_PROMPT, + providerOptions: { + anthropic: { cacheControl: { type: 'ephemeral' } }, + }, + }; + + // Phase F (W7 audit fix) — workspace context message. Sits AFTER + // the main SYSTEM_PROMPT (so the system-prompt cache is unaffected: + // the cache breakpoint is on the static system message; this one + // is small and changes per-turn). The model treats it as + // additional system guidance — "user is currently looking at X" — + // so tool calls like `query_documents` can target the right dataset + // without the user having to repeat it. + // + // Cost: a workspace-context message is typically <150 tokens; the + // cost per turn rounds to nothing. We don't cache it because every + // selection change invalidates the cache anyway. + const contextSystemMessage = + workspaceContext !== null + ? ({ + role: 'system', + content: buildWorkspaceContextPrompt(workspaceContext), + } satisfies ModelMessage) + : null; + + // v6 (2026-05-15, Stream 6.12): convertToModelMessages is now + // async — destructure the awaited array into the prompt. The + // single-line edit the upgrade-inventory doc flagged + // (apps/web/docs/specs/2026-05-15-ai-sdk-v6-upgrade-inventory.md). + const modelMessages = await convertToModelMessages(messages); + // Build a per-request ToolContext so every ctx-aware tool handler + // forwards Cookie + X-XSRF-TOKEN to FastAPI (private-dataset reads + // post Stream 3.1) and emits the same X-Request-Id our telemetry + // uses (Stream 4.5 cross-boundary tracing). Anonymous requests get + // `authHeaders === undefined`; the request id still propagates. + // + // Stream 3.2 extension (2026-05-16): pre-allocate the Voyage usage + // accumulator so `semantic_search_datasets` can increment as it calls + // embedQuery / rerank. Read in onFinish + onError to populate + // chat_usage_events.voyage_embed_tokens + voyage_rerank_units. The + // mutation happens INSIDE the streaming tool loop; reading post- + // stream is safe because all tool calls have completed by then. + const ctx: ToolContext = { + requestId, + voyageUsage: { embedTokens: 0, rerankUnits: 0 }, + }; + const authHeaders = authHeadersFromRequest(req); + if (authHeaders) ctx.authHeaders = authHeaders; + + // Audit 2026-05-20 P1 — track tool-call count per request so + // chat_usage_events.tool_calls_count is populated (pre-fix it was + // hard-coded to 0, breaking per-tool cost rollups). + let toolCallsCount = 0; + const toolNamesSeen = new Set(); + + const result = streamText({ + model: chatModel(), + messages: contextSystemMessage + ? [systemMessage, contextSystemMessage, ...modelMessages] + : [systemMessage, ...modelMessages], + tools: makeTools(ctx), + // Cap output + tool loops to bound cost. See spec §Cost. + // + // maxOutputTokens trajectory: + // 1024 (until 2026-05-14) — too tight. Chatbot accuracy E2E + // audit caught violin-chart fences + // and signal-chart fences being + // truncated mid-stream BEFORE the + // model reaches the ```chart fence. + // The tool succeeds, the + // chart_payload is in the tool + // result, but the model runs out + // of output tokens while composing + // prose and never emits the fence. + // P5 (violin) and P10 (signal) + // from the audit failed this way — + // correct numeric answers, no + // chart rendered. + // 3072 (now) — gives the model enough budget to compose the + // full per-group summary (Saline/CNO stats) AND + // emit the chart fence AND list the Sources + // section. Cost ceiling per output increases + // 3× to ~$0.045/msg output (was $0.015) but + // input remains the binding cost (~$0.04/msg). + // Worst-case overall: ~$0.40/msg vs prior $0.31. + maxOutputTokens: 3072, + // stopWhen replaces v4's `maxSteps`. Cap at 12 model turns so + // deep scientific exploration finishes within one user turn. + // Trajectory of cap bumps: + // 5 (initial) — too tight; "show me a voltage trace" needs to + // find the right binary doc which typically + // requires 4-6 exploratory tool calls before + // fetch_signal is even called + // 8 (Day-4) — multi-tool "what probes in dataset X" worked + // but voltage-trace prompts still ran out of + // steps mid-exploration before reaching + // fetch_signal + // 12 (now) — enough headroom for the full exploration arc: + // semantic_search → get_dataset_class_counts → + // query_documents (probe) → query_documents + // (element) → query_documents + // (daqreader_mfdaq_epochdata_ingested) → + // fetch_signal → compose answer with chart + + // citations. + stopWhen: stepCountIs(12), + temperature: 0.3, + // Audit 2026-05-20 P1 — capture tool-call count + distinct names + // per step so the chat_usage_events row carries real telemetry. + // `onStepFinish` fires once per model turn; we sum tool calls + // across all turns of the request. + onStepFinish: ({ toolCalls }) => { + if (Array.isArray(toolCalls)) { + toolCallsCount += toolCalls.length; + for (const c of toolCalls) { + const name = (c as { toolName?: unknown }).toolName; + if (typeof name === 'string' && name.length > 0) { + toolNamesSeen.add(name); + } + } + } + }, + // The AI SDK's default `maxRetries: 2` (1 initial + 2 retries = + // 3 attempts) with exponential backoff burns up to ~55s of the + // 60s server budget on transient failures before the error + // surfaces to the client. Pre-fix, when Anthropic rate-limited + // upstream the chat would silently stall for the full minute + // before showing the 429. With maxRetries=1, one quick retry + // catches single-shot blips but a hard failure (real rate-limit, + // bad input) surfaces in ~5s. (P1 audit follow-up, 2026-05-14.) + maxRetries: 1, + onError: ({ error }) => { + const e = error instanceof Error ? error : new Error(String(error)); + logEvent('ask.stream.error', { + errorType: e.name, + message: e.message.slice(0, 200), + }); + // Stream 3.2 — record the failure as a usage event so the + // admin cost-dashboard can attribute failed turns. Anthropic + // tokens are zero on a hard error (request didn't bill); we + // still want the row for outcome attribution. Voyage calls + // that completed BEFORE the error counted are still surfaced + // (cost was already incurred — the row would otherwise + // under-report). + const partialUsage = zeroProviderUsage(); + partialUsage.voyageEmbedTokens = ctx.voyageUsage?.embedTokens ?? 0; + partialUsage.voyageRerankUnits = ctx.voyageUsage?.rerankUnits ?? 0; + void logUsage({ + userId, + organizationId: organizationId ?? null, + conversationId: null, + requestId, + startedAt: new Date(askStartedAtMs), + durationMs: Date.now() - askStartedAtMs, + provider: partialUsage, + // Audit 2026-05-20 P1 — capture tool calls that completed + // before the error fired. Pre-fix this was hard-coded to 0. + toolCallsCount, + toolNames: Array.from(toolNamesSeen), + outcome: 'upstream_error', + errorKind: e.name, + modelId: ASK_MODEL_ID, + streamed: true, + }); + }, + onFinish: ({ usage, finishReason }) => { + // Stream 3.2 — happy-path usage event. The AI SDK's + // `usage` callback on streamText returns the aggregated + // token counts across every tool-loop turn for this + // request, mapped here onto the rate-card shape. + void logUsage({ + userId, + organizationId: organizationId ?? null, + conversationId: null, + requestId, + startedAt: new Date(askStartedAtMs), + durationMs: Date.now() - askStartedAtMs, + provider: { + anthropicInputTokens: usage?.inputTokens ?? 0, + anthropicOutputTokens: usage?.outputTokens ?? 0, + anthropicCacheReadTokens: usage?.cachedInputTokens ?? 0, + anthropicCacheCreateTokens: 0, + // Stream 3.2 extension (2026-05-16): Voyage is called inside + // semantic_search_datasets, not through streamText.usage — + // the per-request `ctx.voyageUsage` accumulator captures the + // embed-token totals + per-call rerank-units as each handler + // runs. Read here at the very end of the stream so a multi- + // step turn that calls semantic_search N times gets the + // summed count (every increment is in this single object). + voyageEmbedTokens: ctx.voyageUsage?.embedTokens ?? 0, + voyageRerankUnits: ctx.voyageUsage?.rerankUnits ?? 0, + }, + // Audit 2026-05-20 P1 — populated from onStepFinish so cost + // dashboards can attribute spend per tool. Pre-fix this was + // hard-coded to 0 for every row. + toolCallsCount, + toolNames: Array.from(toolNamesSeen), + outcome: + finishReason === 'stop' || finishReason === 'tool-calls' + ? 'success' + : 'aborted', + modelId: ASK_MODEL_ID, + streamed: true, + }); + }, + }); + + logEvent('ask.stream.start', { ip }); + return result.toUIMessageStreamResponse(); +} + +/** + * Extract the text of the most recent user message for size-only + * logging. Walks the UIMessage parts array (the AI SDK's canonical + * shape) and joins any text-typed parts. Returns '' when no text part + * is found — never throws, never inspects message content beyond + * computing a length. + */ +function lastUserText(messages: UIMessage[]): string { + for (let i = messages.length - 1; i >= 0; i -= 1) { + const m = messages[i]; + if (m?.role !== 'user') continue; + const parts = (m as { parts?: unknown }).parts; + if (!Array.isArray(parts)) return ''; + const texts: string[] = []; + for (const p of parts) { + if (p && typeof p === 'object' && (p as { type?: unknown }).type === 'text') { + const t = (p as { text?: unknown }).text; + if (typeof t === 'string') texts.push(t); + } + } + return texts.join(''); + } + return ''; +} + +function extractMessages(body: unknown): UIMessage[] | null { + if (!body || typeof body !== 'object') return null; + const m = (body as { messages?: unknown }).messages; + if (!Array.isArray(m) || m.length === 0) return null; + // Trust the AI SDK to validate further at convertToModelMessages — + // we just need the array shape OK to forward. + return m as UIMessage[]; +} + +/** + * Audit 2026-05-20 P1 — total user-authored text across the message + * history, used by the per-request size cap. Counts text parts only + * (ignores tool outputs, which we control). + */ +function totalUserTextChars(messages: UIMessage[]): number { + let n = 0; + for (const m of messages) { + const parts = (m as { parts?: unknown }).parts; + if (!Array.isArray(parts)) continue; + for (const p of parts) { + if (p && typeof p === 'object' && (p as { type?: unknown }).type === 'text') { + const t = (p as { text?: unknown }).text; + if (typeof t === 'string') n += t.length; + } + } + } + return n; +} + +/** + * Phase F (W7 audit fix) — workspace context shape the chat client + * sends via `DefaultChatTransport.body.context`. All fields are + * independently optional; absent fields are simply omitted from the + * resulting system prompt. + * + * `selectedXId` keys carry NDI document ids which can be 24-char hex + * ObjectIds, 32-char compound ids, or local NDI identifiers (e.g. + * "NSUBJ-005-PR811") — no shape validation here. The model uses + * these directly as `query_documents` / `walk_provenance` arguments. + */ +interface WorkspaceContext { + datasetId?: string; + datasetName?: string; + selectedSubjectId?: string; + selectedSessionId?: string; + selectedProbeId?: string; + selectedStimulusId?: string; + selectedUnitId?: string; +} + +function extractWorkspaceContext(body: unknown): WorkspaceContext | null { + if (!body || typeof body !== 'object') return null; + const raw = (body as { context?: unknown }).context; + if (!raw || typeof raw !== 'object') return null; + const ctx = raw as Record; + + const result: WorkspaceContext = {}; + const stringKey = (k: keyof WorkspaceContext) => { + const v = ctx[k]; + if (typeof v === 'string' && v.length > 0 && v.length <= 256) { + result[k] = v; + } + }; + stringKey('datasetId'); + stringKey('datasetName'); + stringKey('selectedSubjectId'); + stringKey('selectedSessionId'); + stringKey('selectedProbeId'); + stringKey('selectedStimulusId'); + stringKey('selectedUnitId'); + + return Object.keys(result).length > 0 ? result : null; +} + +/** + * Render the workspace context as a system-message prompt block. + * Kept short — the model already has the full SYSTEM_PROMPT cached; + * this is just situational orientation for the current turn. + * + * The instruction is FRAMED as guidance, not a hard constraint + * ("the user is asking from this context") — leaves the model free + * to redirect when the user actually wants to ask about a different + * dataset. + */ +function buildWorkspaceContextPrompt(ctx: WorkspaceContext): string { + const lines: string[] = ['Workspace context for this turn:']; + if (ctx.datasetName) { + lines.push( + `- Dataset: ${ctx.datasetName}${ + ctx.datasetId ? ` (id: ${ctx.datasetId})` : '' + }`, + ); + } else if (ctx.datasetId) { + lines.push(`- Dataset id: ${ctx.datasetId}`); + } + if (ctx.selectedSubjectId) { + lines.push(`- Selected subject: ${ctx.selectedSubjectId}`); + } + if (ctx.selectedSessionId) { + lines.push(`- Selected session / epoch: ${ctx.selectedSessionId}`); + } + if (ctx.selectedProbeId) { + lines.push(`- Selected probe: ${ctx.selectedProbeId}`); + } + if (ctx.selectedStimulusId) { + lines.push(`- Selected stimulus: ${ctx.selectedStimulusId}`); + } + if (ctx.selectedUnitId) { + lines.push(`- Selected unit (vmspikesummary): ${ctx.selectedUnitId}`); + } + lines.push(''); + lines.push( + 'Treat this as default scope: when the user asks "this dataset" / "this subject" / "the current session", they mean the values above. If they explicitly name a different dataset/subject/etc., the explicit reference wins.', + ); + return lines.join('\n'); +} diff --git a/apps/web/app/api/cron/dataset-health/route.ts b/apps/web/app/api/cron/dataset-health/route.ts new file mode 100644 index 00000000..8b069ee0 --- /dev/null +++ b/apps/web/app/api/cron/dataset-health/route.ts @@ -0,0 +1,202 @@ +/** + * GET /api/cron/dataset-health — nightly Dataset Health snapshot. + * + * Stream 6.8 (2026-05-15). Iterates every published dataset, fetches + * the rich summary + class-counts, runs the full invariant set + * (`apps/web/lib/data-quality/invariants.ts`), and persists violations + * to the `dataset_health_violations` table. The admin page at + * `/admin/data-health` (Stream 6.9) reads from that table; the catalog + * badge (Stream 6.10) shows compact-safe checks today and will gain + * the full set once we wire it to read from the table. + * + * Vercel Cron schedule: configured in vercel.json. Trigger guards: + * + * - `Authorization: Bearer ${CRON_SECRET}` for external callers + * - `x-vercel-cron: 1` for Vercel-managed cron (set at the edge) + * + * Returns a JSON summary of the scan so the cron-run logs surface + * the per-dataset outcome at a glance. + */ +import { NextResponse, type NextRequest } from 'next/server'; + +import { env } from '@/lib/env'; +import { logEvent } from '@/lib/ndi/tools/shared'; +import { isProductionEnv } from '@/lib/runtime-env'; +import { + checkDatasetHealth, + type DatasetSummaryFacts, +} from '@/lib/data-quality/invariants'; +import { replaceViolationsForDataset } from '@/lib/data-quality/persistence'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; +// The scan iterates all published datasets sequentially (~8 today, +// ~50 within a year). Single dataset summary fetch takes ~1-3s on a +// cold cache. 60s is the sweet spot — long enough to scan ~20 cold +// datasets, short enough to fail fast on a wedged backend. +export const maxDuration = 60; + +interface CronSummary { + datasets_scanned: number; + datasets_with_violations: number; + total_violations: number; + failures: Array<{ dataset_id: string; reason: string }>; +} + +function authorize(req: NextRequest): boolean { + // Vercel cron sets x-vercel-cron: 1 at the edge. + if (req.headers.get('x-vercel-cron') === '1') return true; + // External callers (manual trigger from CI / a script) must echo + // the CRON_SECRET as a Bearer. + const secret = env.CRON_SECRET; + if (!secret) return false; + const auth = req.headers.get('authorization') ?? ''; + if (!auth.startsWith('Bearer ')) return false; + return auth.slice('Bearer '.length).trim() === secret; +} + +function baseUrl(): string | null { + if (env.VERCEL_GIT_COMMIT_REF === 'feat/experimental-ask-chat') { + return 'https://ndb-v2-experimental.up.railway.app'; + } + const u = env.INTERNAL_API_URL; + return typeof u === 'string' && u.length > 0 ? u : null; +} + +interface PublishedDatasetLite { + id?: string; + _id?: string; + name?: string; +} + +interface BackendCounts { + totalDocuments?: number; + counts?: { + sessions?: number; + subjects?: number; + probes?: number; + elements?: number; + epochs?: number; + totalDocuments?: number; + }; + classCounts?: Record; + species?: Array<{ label?: string }> | null; + brainRegions?: Array<{ label?: string }> | null; + strains?: Array<{ label?: string }> | null; +} + +async function fetchJson(url: string): Promise { + try { + const res = await fetch(url, { cache: 'no-store' }); + if (!res.ok) return null; + return (await res.json()) as T; + } catch { + return null; + } +} + +export async function GET(req: NextRequest) { + if (!authorize(req)) { + return NextResponse.json({ error: 'unauthorized' }, { status: 401 }); + } + // Audit 2026-05-20 P0 #4 — Vercel project-level crons fire against + // every active deployment INCLUDING Preview. Pre-fix, the Preview + // deploy's nightly snapshot was overwriting production rows in the + // shared Postgres tables. No-op on non-production deploys. + if (!isProductionEnv()) { + logEvent('dataset_health.cron.skipped_non_production', { + env: process.env.VERCEL_ENV ?? 'unknown', + }); + return NextResponse.json({ + ok: true, + skipped: 'non-production env', + env: process.env.VERCEL_ENV ?? 'unknown', + }); + } + const base = baseUrl(); + if (!base) { + return NextResponse.json( + { error: 'catalog_service_not_configured' }, + { status: 503 }, + ); + } + + const summary: CronSummary = { + datasets_scanned: 0, + datasets_with_violations: 0, + total_violations: 0, + failures: [], + }; + + // 1. Fetch every published dataset's id+name. + // pageSize=100 covers our catalog comfortably; a follow-up adds + // pagination if we ever exceed it. + const published = await fetchJson<{ + datasets?: PublishedDatasetLite[]; + }>(`${base}/api/datasets/published?page=1&pageSize=100`); + const datasets = published?.datasets ?? []; + if (datasets.length === 0) { + logEvent('dataset_health.cron.no_datasets', {}); + return NextResponse.json(summary); + } + + // 2. Per-dataset: fetch summary + class-counts, build facts, check + // invariants, persist. Sequential to keep upstream load light; + // can parallel-batch later if the scan exceeds maxDuration. + for (const ds of datasets) { + const id = ds.id ?? ds._id; + if (typeof id !== 'string' || id.length === 0) continue; + + const [datasetSummary, classCounts] = await Promise.all([ + fetchJson(`${base}/api/datasets/${id}/summary`), + fetchJson(`${base}/api/datasets/${id}/class-counts`), + ]); + if (!datasetSummary && !classCounts) { + summary.failures.push({ dataset_id: id, reason: 'upstream_unreachable' }); + continue; + } + const facts: DatasetSummaryFacts = { + datasetId: id, + datasetName: ds.name ?? id, + species: (datasetSummary?.species ?? []).map((s) => s.label ?? ''), + brainRegions: (datasetSummary?.brainRegions ?? []).map( + (r) => r.label ?? '', + ), + strains: (datasetSummary?.strains ?? []).map((s) => s.label ?? ''), + totalDocuments: + datasetSummary?.counts?.totalDocuments ?? + classCounts?.totalDocuments ?? + 0, + classCounts: classCounts?.classCounts ?? {}, + derivedCounts: { + sessions: datasetSummary?.counts?.sessions ?? 0, + subjects: datasetSummary?.counts?.subjects ?? 0, + elements: datasetSummary?.counts?.elements ?? 0, + epochs: datasetSummary?.counts?.epochs ?? 0, + probes: datasetSummary?.counts?.probes ?? 0, + }, + }; + const violations = checkDatasetHealth(facts); + try { + await replaceViolationsForDataset(id, ds.name ?? null, violations); + } catch (err) { + summary.failures.push({ + dataset_id: id, + reason: + err instanceof Error ? err.message : 'persistence_failure', + }); + continue; + } + summary.datasets_scanned += 1; + summary.total_violations += violations.length; + if (violations.length > 0) summary.datasets_with_violations += 1; + } + + logEvent('dataset_health.cron.complete', { + datasets_scanned: summary.datasets_scanned, + datasets_with_violations: summary.datasets_with_violations, + total_violations: summary.total_violations, + failure_count: summary.failures.length, + }); + return NextResponse.json(summary); +} diff --git a/apps/web/app/api/cron/warm-cache/route.ts b/apps/web/app/api/cron/warm-cache/route.ts index 727df0bd..20dcef6d 100644 --- a/apps/web/app/api/cron/warm-cache/route.ts +++ b/apps/web/app/api/cron/warm-cache/route.ts @@ -41,6 +41,7 @@ import { NextResponse } from 'next/server'; import { env } from '@/lib/env'; +import { isProductionEnv } from '@/lib/runtime-env'; export const runtime = 'nodejs'; export const dynamic = 'force-dynamic'; @@ -171,6 +172,18 @@ export async function GET(req: Request) { return new NextResponse('unauthorized', { status: 401 }); } + // Audit 2026-05-20 P0 #4 — Vercel project-level crons fire against + // every active deployment INCLUDING Preview. Pre-fix, the Preview + // deploy of this branch was re-warming the preview's edge cache + // every 5 minutes and burning Vercel function invocations for no + // user benefit. No-op on non-production deploys. + if (!isProductionEnv()) { + return NextResponse.json( + { ok: true, skipped: 'non-production env', env: process.env.VERCEL_ENV ?? 'unknown' }, + { headers: { 'Cache-Control': 'no-store' } }, + ); + } + const origin = req.headers.get('host') ? `https://${req.headers.get('host')}` : env.VERCEL_URL diff --git a/apps/web/app/api/datasets/[id]/cross-table-query/route.ts b/apps/web/app/api/datasets/[id]/cross-table-query/route.ts new file mode 100644 index 00000000..b890321f --- /dev/null +++ b/apps/web/app/api/datasets/[id]/cross-table-query/route.ts @@ -0,0 +1,68 @@ +/** + * POST /api/datasets/[id]/cross-table-query — workspace panel endpoint. + * + * Thin route handler that reuses the chat-side + * `crossTableQueryHandler` (lib/ndi/tools/cross-table-query.ts) so the + * BehavioralCompare panel's Cross-table mode and the chat's + * `cross_table_query` tool render identical pair sets + chart + * payloads off the same code path (ADR-002). + * + * Mirrors `tabular-query/route.ts`'s pattern: + * - Threads auth headers via toolContextFromRequest (ADR-003) + * - Threads inbound x-request-id through to FastAPI for tracing + * (ADR-005) + * - Surfaces the full chat-tool envelope (pair_count, unjoined, + * group_summary, chart_payload, references, empty_hint) so the + * panel and chat see the same shape + * + * Path-id guard rejects anything that isn't bare alphanumeric/_- so + * a crafted path can't reach an unintended upstream URL. + */ +import { type NextRequest } from 'next/server'; + +import { + crossTableQueryHandler, + crossTableQueryInput, +} from '@/lib/ndi/tools/cross-table-query'; +import { toolContextFromRequest } from '@/lib/ndi/tools/shared'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +interface RouteContext { + params: Promise<{ id: string }>; +} + +export async function POST(req: NextRequest, { params }: RouteContext) { + const { id } = await params; + if (!/^[a-zA-Z0-9_-]+$/.test(id)) { + return Response.json({ error: 'invalid_dataset_id' }, { status: 400 }); + } + + let body: unknown; + try { + body = await req.json(); + } catch { + return Response.json({ error: 'invalid_json_body' }, { status: 400 }); + } + + // URL wins on collision — the path id is the canonical resource id. + const merged = + body && typeof body === 'object' + ? { ...(body as Record), datasetId: id } + : { datasetId: id }; + + const parsed = crossTableQueryInput.safeParse(merged); + if (!parsed.success) { + return Response.json( + { error: 'invalid_input', detail: parsed.error.message }, + { status: 400 }, + ); + } + + const result = await crossTableQueryHandler( + parsed.data, + toolContextFromRequest(req), + ); + return Response.json(result, { status: 200 }); +} diff --git a/apps/web/app/api/datasets/[id]/documents/[docId]/signal/route.ts b/apps/web/app/api/datasets/[id]/documents/[docId]/signal/route.ts new file mode 100644 index 00000000..c64d51ab --- /dev/null +++ b/apps/web/app/api/datasets/[id]/documents/[docId]/signal/route.ts @@ -0,0 +1,194 @@ +/** + * GET /api/datasets/[id]/documents/[docId]/signal — workspace panel + * timeseries endpoint. Transparent JSON proxy to FastAPI with: + * + * 1. Path-param allowlist regex (datasetId, docId) + * 2. Query-param zod validation (downsample, t0, t1, file) + * 3. Auth headers forwarded via toolContextFromRequest + * 4. X-Request-Id propagated for cross-boundary tracing + * + * Audit 2026-05-20 P1 follow-up: pre-fix, three workspace panels + * (SignalChart, TrajectoryChart inside BehavioralTrackPanel, the + * inline raster query inside PatchClampStepFamilyPanel) and the + * chart-fence-rendered SignalChart in /ask all fetched this URL via + * the Vercel rewrite fallthrough to FastAPI. The rewrite forwards + * cookies transparently — so auth worked — but there was no + * `X-Request-Id` propagation for cross-boundary tracing, no + * Next-layer input validation, and the panel pattern diverged from + * the 5 other wrapper routes (psth, spike-summary, tabular-query, + * treatment-timeline, cross-table-query). This route closes that + * gap. + * + * Unlike the other 5 wrapper routes, this one does NOT delegate to a + * tool handler. The `fetch_signal` tool handler in + * `lib/ndi/tools/fetch-signal.ts` projects the backend response down + * to a leaner LLM-facing shape — strips the data arrays and exposes + * only counts + chart metadata — to keep the context window small. + * The workspace chart NEEDS the full arrays for rendering, so we + * pass the upstream JSON through verbatim and let the SignalChart + * client-side renderer consume it directly. + * + * Binary endpoints (`/data/image`, `/data/video`, `/data/timeseries`, + * etc. under `lib/api/binary.ts`) intentionally stay on the Vercel + * rewrite fallthrough — they're pass-through binary streams where + * the rewrite is the right pattern (no Node hop, Vercel CDN-friendly, + * lower latency for multi-MB blobs). Auth forwarding works the same + * via transparent cookie proxy. + */ +import { type NextRequest } from 'next/server'; +import { z } from 'zod'; + +import { + baseUrl, + freshRequestId, + logEvent, + toolContextFromRequest, +} from '@/lib/ndi/tools/shared'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +interface RouteContext { + params: Promise<{ id: string; docId: string }>; +} + +// Path-param allowlist matches the other 5 wrapper routes. NDI ids +// are alphanumeric + underscore + hyphen; anything else is a 400. +const PATH_ID_REGEX = /^[a-zA-Z0-9_-]+$/; + +// Mirrors `fetchSignalInput` in `lib/ndi/tools/fetch-signal.ts` for +// the query params we forward. Keeping the two validators in lockstep +// means a request that passes here also passes the tool-layer schema, +// so the same backend contract holds for chat and workspace callers. +const QuerySchema = z.object({ + downsample: z.preprocess( + (v) => (typeof v === 'string' && v.length > 0 ? Number(v) : undefined), + z.number().int().positive().min(10).max(5000).optional(), + ), + t0: z.preprocess( + (v) => (typeof v === 'string' && v.length > 0 ? Number(v) : undefined), + z.number().optional(), + ), + t1: z.preprocess( + (v) => (typeof v === 'string' && v.length > 0 ? Number(v) : undefined), + z.number().optional(), + ), + file: z + .string() + .min(1) + .max(64) + .regex(/^[A-Za-z0-9_.-]+$/, 'file must be a bare filename (alnum + _ . -)') + .optional(), +}); + +export interface SignalWrapperDeps { + /** Inject `fetch` for tests. Defaults to the global. */ + fetchFn?: typeof fetch; +} + +/** + * Internal handler exported for tests. Same pattern as + * `handlePost` in the GitHub Template routes — Next.js doesn't allow + * extra params on a route export, so the public `GET` below delegates + * with no injected deps. + */ +export async function handleGet( + req: NextRequest, + ctxParams: { id: string; docId: string }, + deps: SignalWrapperDeps = {}, +): Promise { + const { id, docId } = ctxParams; + if (!PATH_ID_REGEX.test(id)) { + return Response.json({ error: 'invalid_dataset_id' }, { status: 400 }); + } + if (!PATH_ID_REGEX.test(docId)) { + return Response.json({ error: 'invalid_doc_id' }, { status: 400 }); + } + + const url = new URL(req.url); + const queryInput = { + downsample: url.searchParams.get('downsample') ?? undefined, + t0: url.searchParams.get('t0') ?? undefined, + t1: url.searchParams.get('t1') ?? undefined, + file: url.searchParams.get('file') ?? undefined, + }; + const parsed = QuerySchema.safeParse(queryInput); + if (!parsed.success) { + return Response.json( + { error: 'invalid_query', detail: parsed.error.message }, + { status: 400 }, + ); + } + + const base = baseUrl(); + if (!base) { + return Response.json({ error: 'service_not_configured' }, { status: 503 }); + } + + const ctx = toolContextFromRequest(req); + const requestId = ctx.requestId ?? freshRequestId(); + + const qs = new URLSearchParams(); + if (parsed.data.downsample !== undefined) { + qs.set('downsample', String(parsed.data.downsample)); + } + if (parsed.data.t0 !== undefined) qs.set('t0', String(parsed.data.t0)); + if (parsed.data.t1 !== undefined) qs.set('t1', String(parsed.data.t1)); + if (parsed.data.file !== undefined) qs.set('file', parsed.data.file); + + const upstreamUrl = + `${base}/api/datasets/${encodeURIComponent(id)}` + + `/documents/${encodeURIComponent(docId)}/signal` + + (qs.toString() ? `?${qs.toString()}` : ''); + + const fetchFn = deps.fetchFn ?? fetch; + const start = Date.now(); + let upstream: Response; + try { + upstream = await fetchFn(upstreamUrl, { + method: 'GET', + headers: { + Accept: 'application/json', + 'X-Request-Id': requestId, + ...(ctx.authHeaders ?? {}), + }, + cache: 'no-store', + }); + } catch (err) { + logEvent('workspace.signal.upstream_error', { + datasetId: id, + docId, + requestId, + durationMs: Date.now() - start, + errorKind: err instanceof Error ? err.name : 'unknown', + }); + return Response.json({ error: 'upstream_unreachable' }, { status: 502 }); + } + + logEvent('workspace.signal.fetched', { + datasetId: id, + docId, + requestId, + upstreamStatus: upstream.status, + durationMs: Date.now() - start, + }); + + // Transparent JSON pass-through. Preserve the upstream status code + // (404/422/500/etc.) so the chart can branch on it the same way the + // pre-fix Vercel-rewrite path did. Strip cookies + cache-control — + // workspace data is per-user, never cacheable at the browser layer. + const body = await upstream.text(); + return new Response(body, { + status: upstream.status, + headers: { + 'content-type': + upstream.headers.get('content-type') ?? 'application/json', + 'cache-control': 'no-store', + }, + }); +} + +export async function GET(req: NextRequest, { params }: RouteContext) { + const resolved = await params; + return handleGet(req, resolved); +} diff --git a/apps/web/app/api/datasets/[id]/psth/route.ts b/apps/web/app/api/datasets/[id]/psth/route.ts new file mode 100644 index 00000000..ab2c5ea1 --- /dev/null +++ b/apps/web/app/api/datasets/[id]/psth/route.ts @@ -0,0 +1,60 @@ +/** + * POST /api/datasets/[id]/psth — workspace panel endpoint. + * + * Thin route handler that reuses the chat-side `psthHandler` + * (lib/ndi/tools/psth.ts). Same pattern as spike-summary: workspace + * panel hits this route, route forwards the caller's auth headers, + * handler reaches Railway server-side via `baseUrl()`. + */ +import { type NextRequest } from 'next/server'; + +import { psthHandler, psthInput } from '@/lib/ndi/tools/psth'; +import { toolContextFromRequest } from '@/lib/ndi/tools/shared'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +interface RouteContext { + params: Promise<{ id: string }>; +} + +export async function POST(req: NextRequest, { params }: RouteContext) { + const { id } = await params; + if (!/^[a-zA-Z0-9_-]+$/.test(id)) { + return Response.json({ error: 'invalid_dataset_id' }, { status: 400 }); + } + + let body: unknown; + try { + body = await req.json(); + } catch { + return Response.json({ error: 'invalid_json_body' }, { status: 400 }); + } + + // Merge the route param into the body so the handler's zod schema + // sees `datasetId`. URL wins on collision — it's the canonical + // resource identifier. + const merged = + body && typeof body === 'object' + ? { ...(body as Record), datasetId: id } + : { datasetId: id }; + + const parsed = psthInput.safeParse(merged); + if (!parsed.success) { + return Response.json( + { error: 'invalid_input', detail: parsed.error.message }, + { status: 400 }, + ); + } + + // toolContextFromRequest threads both auth headers AND the + // inbound `x-request-id` (or Vercel's `x-vercel-id`) through to + // the handler so the FastAPI proxy can correlate this call with + // the rest of the user's panel-load trace. See ADR-005 + + // `apps/web/docs/operations/three-surfaces.md`. + const result = await psthHandler(parsed.data, toolContextFromRequest(req)); + // Handler returns either a `ToolError` (`{ error: string }`) or a + // `PsthToolResult` envelope. Both shapes pass through verbatim — + // the panel discriminates on the presence of `error`. + return Response.json(result, { status: 200 }); +} diff --git a/apps/web/app/api/datasets/[id]/spike-summary/route.ts b/apps/web/app/api/datasets/[id]/spike-summary/route.ts new file mode 100644 index 00000000..d35c7064 --- /dev/null +++ b/apps/web/app/api/datasets/[id]/spike-summary/route.ts @@ -0,0 +1,80 @@ +/** + * POST /api/datasets/[id]/spike-summary — workspace panel endpoint. + * + * Thin route handler that reuses the chat-side `fetchSpikeSummaryHandler` + * (lib/ai/tools/fetch-spike-summary.ts). The chat path invokes the + * handler from the Anthropic streamText tool loop; the workspace panel + * invokes the same handler over HTTP so the GUI gets identical chart + * payloads + references the chat would produce. + * + * This route takes precedence over the catch-all `/api/:path*` rewrite + * in `next.config.ts` (Next.js resolves `app/api/` route handlers + * before falling through to rewrites), so the FastAPI never sees this + * path — the handler itself reaches Railway server-side via + * `baseUrl()` exactly like the chat tool does. That keeps the chat / + * panel parity tight: one path of code does the discovery, filtering, + * stride-sampling, and payload shaping. + * + * Path-id guard mirrors `/api/datasets/[id]/route.ts` — accept only + * the bare alphanumeric/_- id shapes Mongo uses, so a crafted path + * can't reach an unintended upstream URL. + */ +import { type NextRequest } from 'next/server'; + +import { + fetchSpikeSummaryHandler, + fetchSpikeSummaryInput, +} from '@/lib/ndi/tools/fetch-spike-summary'; +import { toolContextFromRequest } from '@/lib/ndi/tools/shared'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +interface RouteContext { + params: Promise<{ id: string }>; +} + +export async function POST(req: NextRequest, { params }: RouteContext) { + const { id } = await params; + if (!/^[a-zA-Z0-9_-]+$/.test(id)) { + return Response.json({ error: 'invalid_dataset_id' }, { status: 400 }); + } + + let body: unknown; + try { + body = await req.json(); + } catch { + return Response.json({ error: 'invalid_json_body' }, { status: 400 }); + } + + // Merge the route param into the body so the handler's zod schema + // sees `datasetId`. We accept either spelling defensively — if the + // client supplied a different id in the body, the URL wins (the URL + // is the canonical resource identifier). + const merged = + body && typeof body === 'object' + ? { ...(body as Record), datasetId: id } + : { datasetId: id }; + + const parsed = fetchSpikeSummaryInput.safeParse(merged); + if (!parsed.success) { + return Response.json( + { error: 'invalid_input', detail: parsed.error.message }, + { status: 400 }, + ); + } + + // toolContextFromRequest threads both auth headers (Cookie + + // X-XSRF-TOKEN — workspace panels are auth-gated) AND the + // inbound `x-request-id` / `x-vercel-id` so cross-boundary tracing + // can stitch the user's panel load with the FastAPI log lines. + // See ADR-005 + `apps/web/docs/operations/three-surfaces.md`. + const result = await fetchSpikeSummaryHandler( + parsed.data, + toolContextFromRequest(req), + ); + // The handler returns either a `ToolError` (`{ error: string }`) or + // a `FetchSpikeSummaryToolResult` envelope. Both shapes are returned + // verbatim — the panel discriminates on the presence of `error`. + return Response.json(result, { status: 200 }); +} diff --git a/apps/web/app/api/datasets/[id]/tables/[className]/route.ts b/apps/web/app/api/datasets/[id]/tables/[className]/route.ts index a6d91a9b..5b942970 100644 --- a/apps/web/app/api/datasets/[id]/tables/[className]/route.ts +++ b/apps/web/app/api/datasets/[id]/tables/[className]/route.ts @@ -31,7 +31,17 @@ interface RouteContext { params: Promise<{ id: string; className: string }>; } -export async function GET(_req: NextRequest, { params }: RouteContext) { +/** + * Forward `page` + `pageSize` so each pagination slice gets its own + * cache key. Audit 2026-05-18 finding B1 caught us discarding query + * params here — Stream 5.8's whole `usePagedDatasetTable` pagination + * was silently falling through to the legacy unpaged envelope, which + * meant the ~95% egress saving the spec promised never landed for + * traffic flowing through this proxy. Mirror the documents/route.ts + * pattern: only forward params the backend actually reads, so bonus + * params (analytics tracking, etc.) don't needlessly fragment cache. + */ +export async function GET(req: NextRequest, { params }: RouteContext) { const { id, className } = await params; if (!/^[a-zA-Z0-9_-]+$/.test(id) || !/^[a-zA-Z0-9_-]+$/.test(className)) { return new Response( @@ -45,5 +55,16 @@ export async function GET(_req: NextRequest, { params }: RouteContext) { }, ); } - return cachedProxy(`/api/datasets/${id}/tables/${className}`, CACHE_ITEM); + + const url = new URL(req.url); + const params_q = new URLSearchParams(); + const page = url.searchParams.get('page'); + const pageSize = url.searchParams.get('pageSize'); + if (page) params_q.set('page', page); + if (pageSize) params_q.set('pageSize', pageSize); + const qs = params_q.toString(); + const path = qs + ? `/api/datasets/${id}/tables/${className}?${qs}` + : `/api/datasets/${id}/tables/${className}`; + return cachedProxy(path, CACHE_ITEM); } diff --git a/apps/web/app/api/datasets/[id]/tabular-query/route.ts b/apps/web/app/api/datasets/[id]/tabular-query/route.ts new file mode 100644 index 00000000..8b29b806 --- /dev/null +++ b/apps/web/app/api/datasets/[id]/tabular-query/route.ts @@ -0,0 +1,78 @@ +/** + * POST /api/datasets/[id]/tabular-query — workspace panel endpoint. + * + * Thin route handler that reuses the chat-side `tabularQueryHandler` + * (lib/ndi/tools/tabular-query.ts) so the BehavioralCompare panel and + * the chat's `tabular_query` tool render identical group statistics + * and chart payloads off the same code path (ADR-002). + * + * Migration note (Stream 4.1, 2026-05-15): BehavioralComparePanel + * previously bypassed this wrapper, calling + * `GET /api/datasets/:id/tabular_query` (the underscore-spelled + * FastAPI path) directly via the Vercel rewrite. That worked for + * public datasets (GET is exempt from CSRF) but skipped the + * cross-boundary tracing + auth-forwarding contract every other + * mutation panel honors. Switching to this POST wrapper: + * + * - Threads auth headers via toolContextFromRequest (ADR-003) + * - Threads the inbound x-request-id through to FastAPI for + * cross-boundary tracing (ADR-005) + * - Surfaces the full chat-tool envelope (groups_summary with + * mean/median/std/min/max/q1/q3 + chart_payload + references + + * empty_hint) instead of a custom intermediate shape + * + * Path-id guard mirrors the sibling wrapper routes — accept only the + * bare alphanumeric/_- id shapes Mongo uses, so a crafted path can't + * reach an unintended upstream URL. + */ +import { type NextRequest } from 'next/server'; + +import { + tabularQueryHandler, + tabularQueryInput, +} from '@/lib/ndi/tools/tabular-query'; +import { toolContextFromRequest } from '@/lib/ndi/tools/shared'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +interface RouteContext { + params: Promise<{ id: string }>; +} + +export async function POST(req: NextRequest, { params }: RouteContext) { + const { id } = await params; + if (!/^[a-zA-Z0-9_-]+$/.test(id)) { + return Response.json({ error: 'invalid_dataset_id' }, { status: 400 }); + } + + let body: unknown; + try { + body = await req.json(); + } catch { + return Response.json({ error: 'invalid_json_body' }, { status: 400 }); + } + + // URL wins on collision — the path id is the canonical resource id. + const merged = + body && typeof body === 'object' + ? { ...(body as Record), datasetId: id } + : { datasetId: id }; + + const parsed = tabularQueryInput.safeParse(merged); + if (!parsed.success) { + return Response.json( + { error: 'invalid_input', detail: parsed.error.message }, + { status: 400 }, + ); + } + + const result = await tabularQueryHandler( + parsed.data, + toolContextFromRequest(req), + ); + // The handler returns either a `ToolError` (`{ error: string }`) or + // a `TabularQueryToolResult` envelope. Both shapes are returned + // verbatim — the panel discriminates on the presence of `error`. + return Response.json(result, { status: 200 }); +} diff --git a/apps/web/app/api/datasets/[id]/treatment-timeline/route.ts b/apps/web/app/api/datasets/[id]/treatment-timeline/route.ts new file mode 100644 index 00000000..f72481f8 --- /dev/null +++ b/apps/web/app/api/datasets/[id]/treatment-timeline/route.ts @@ -0,0 +1,75 @@ +/** + * POST /api/datasets/[id]/treatment-timeline — workspace panel endpoint. + * + * Thin route handler that reuses the chat-side `treatmentTimelineHandler` + * (lib/ndi/tools/treatment-timeline.ts). Same parity contract as the + * spike-summary wrapper: chat invokes the handler from the Anthropic + * streamText tool loop; the workspace panel invokes the same handler + * over HTTP so the GUI gets identical chart payloads + references the + * chat would produce. + * + * Auth-forwarding: the workspace is auth-gated, so every request that + * lands here carries the user's session Cookie + X-XSRF-TOKEN. We + * extract both and pass them via `ToolContext` to the handler so its + * outbound FastAPI calls authenticate the caller and return private- + * dataset rows the user has access to. + * + * Path-id guard mirrors `/api/datasets/[id]/route.ts` — accept only + * the bare alphanumeric/_- id shapes Mongo uses, so a crafted path + * can't reach an unintended upstream URL. + */ +import { type NextRequest } from 'next/server'; + +import { + treatmentTimelineHandler, + treatmentTimelineInput, +} from '@/lib/ndi/tools/treatment-timeline'; +import { toolContextFromRequest } from '@/lib/ndi/tools/shared'; + +export const runtime = 'nodejs'; +export const dynamic = 'force-dynamic'; + +interface RouteContext { + params: Promise<{ id: string }>; +} + +export async function POST(req: NextRequest, { params }: RouteContext) { + const { id } = await params; + if (!/^[a-zA-Z0-9_-]+$/.test(id)) { + return Response.json({ error: 'invalid_dataset_id' }, { status: 400 }); + } + + let body: unknown; + try { + body = await req.json(); + } catch { + return Response.json({ error: 'invalid_json_body' }, { status: 400 }); + } + + // Merge the route param into the body so the handler's zod schema + // sees `datasetId` even when the client only supplied the URL path. + const merged = + body && typeof body === 'object' + ? { ...(body as Record), datasetId: id } + : { datasetId: id }; + + const parsed = treatmentTimelineInput.safeParse(merged); + if (!parsed.success) { + return Response.json( + { error: 'invalid_input', detail: parsed.error.message }, + { status: 400 }, + ); + } + + // toolContextFromRequest threads auth headers + the inbound + // request id so cross-boundary tracing can correlate this call + // with the FastAPI log lines for the same panel load. + const result = await treatmentTimelineHandler( + parsed.data, + toolContextFromRequest(req), + ); + // The handler returns either a `ToolError` (`{ error: string }`) or + // a `TreatmentTimelineResult` envelope. Both shapes are returned + // verbatim — the panel discriminates on the presence of `error`. + return Response.json(result, { status: 200 }); +} diff --git a/apps/web/app/api/github/create-analysis-repo/route.ts b/apps/web/app/api/github/create-analysis-repo/route.ts new file mode 100644 index 00000000..03f10c7d --- /dev/null +++ b/apps/web/app/api/github/create-analysis-repo/route.ts @@ -0,0 +1,300 @@ +/** + * POST /api/github/create-analysis-repo — derives a new private GitHub + * repo for the authenticated user from + * `Waltham-Data-Science/ndi-analysis-template`, then commits the + * panel-specific `current_analysis.py` into it (ADR-010). + * + * Flow: + * 1. Validate env: GITHUB_CLIENT_ID + GITHUB_CLIENT_SECRET set → + * feature is configured. If not, 503 `feature_not_configured`. + * 2. Resolve the user's GitHub OAuth token from the cookie. Missing + * → 401 `github_auth_required` (client will kick off /api/github/oauth/start). + * 3. Validate the request body (zod). Bad shape → 400 `invalid_input`. + * 4. Slug a candidate repo name; check collisions in the user's + * namespace; suffix `-2`, `-3` up to 5 attempts. + * 5. Call `octokit.rest.repos.createUsingTemplate({...})`. The new + * repo is private and only owned by this user (we never push to + * Waltham-Data-Science). + * 6. Poll `GET /repos/{owner}/{repo}` until the repo is provisioned + * (max 10 attempts × 500ms; GitHub typically returns it in <2s). + * 7. Generate `current_analysis.py` via `generateCurrentAnalysis` + * and commit it via `createOrUpdateFileContents`. + * 8. Return `{ url, name, owner }`. + * + * Error envelopes use the `GithubErrorEnvelope` type so the client + * can branch on `code` without dotted paths. + */ +import { NextResponse } from 'next/server'; +import { Octokit } from '@octokit/rest'; + +import { generateCurrentAnalysis } from '@/lib/ndi/code-export/current-analysis'; +import { env } from '@/lib/env'; +import { getGitHubTokenFromRequest } from '@/lib/github/oauth'; +import { buildRepoSlug, withCollisionSuffix } from '@/lib/github/slug'; +import { + GithubAnalysisRequestSchema, + TEMPLATE_OWNER, + TEMPLATE_REPO, + type GithubErrorEnvelope, +} from '@/lib/github/types'; +import { logEvent } from '@/lib/ndi/tools/shared'; + +export const runtime = 'nodejs'; +export const maxDuration = 60; + +const MAX_COLLISION_ATTEMPTS = 5; +const POLL_INTERVAL_MS = 500; +const POLL_MAX_ATTEMPTS = 10; + +export interface OctokitDeps { + /** Inject an Octokit factory for tests. Defaults to the real constructor. */ + buildOctokit?: (token: string) => Octokit; + /** Inject a delay for tests (default node setTimeout). */ + delay?: (ms: number) => Promise; +} + +function jsonError( + status: number, + body: GithubErrorEnvelope, +): NextResponse { + return NextResponse.json(body, { status }); +} + +async function defaultDelay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Find the first free repo name for `username`. Returns the chosen + * name or null if all MAX_COLLISION_ATTEMPTS were taken (extremely + * unlikely — the date suffix already pre-disambiguates). + */ +async function pickAvailableRepoName( + octokit: Octokit, + username: string, + baseSlug: string, +): Promise { + for (let i = 1; i <= MAX_COLLISION_ATTEMPTS; i++) { + const candidate = withCollisionSuffix(baseSlug, i); + try { + await octokit.rest.repos.get({ owner: username, repo: candidate }); + // 200 → repo exists, try the next suffix. + } catch (err) { + if (err instanceof Error && 'status' in err && (err as { status: number }).status === 404) { + return candidate; + } + // Any other error (rate-limit, 401, 5xx) bubbles up. + throw err; + } + } + return null; +} + +/** + * Poll the new repo until GitHub confirms it's ready. `createUsingTemplate` + * returns 201 immediately but the repo isn't necessarily clonable for + * up to a few seconds. Without this we sometimes saw 404 on the first + * `createOrUpdateFileContents`. + */ +async function pollUntilReady( + octokit: Octokit, + owner: string, + repo: string, + delay: (ms: number) => Promise, +): Promise { + for (let i = 0; i < POLL_MAX_ATTEMPTS; i++) { + try { + const { data } = await octokit.rest.repos.get({ owner, repo }); + if (data.created_at) return true; + } catch { + // 404 while GitHub is provisioning — keep polling. + } + await delay(POLL_INTERVAL_MS); + } + return false; +} + +/** + * Internal handler exported for tests. The actual `POST` export below + * delegates here with no injected deps — Next.js doesn't allow extra + * params on a route export. + */ +export async function handlePost( + req: Request, + deps: OctokitDeps = {}, +): Promise { + const clientId = env.GITHUB_CLIENT_ID; + const clientSecret = env.GITHUB_CLIENT_SECRET; + if (!clientId || !clientSecret) { + return jsonError(503, { + error: 'feature_not_configured', + message: + 'GitHub integration is not configured. Contact ops to enable it.', + }); + } + + const token = getGitHubTokenFromRequest(req); + if (!token) { + return jsonError(401, { + error: 'github_auth_required', + message: 'Connect your GitHub account first.', + }); + } + + let body: unknown; + try { + body = await req.json(); + } catch { + return jsonError(400, { + error: 'invalid_input', + message: 'Request body must be valid JSON.', + }); + } + + const parsed = GithubAnalysisRequestSchema.safeParse(body); + if (!parsed.success) { + return jsonError(400, { + error: 'invalid_input', + message: 'Request body failed validation.', + details: { issues: parsed.error.issues }, + }); + } + + const { panelState, datasetName, question } = parsed.data; + + const buildOctokit = deps.buildOctokit ?? ((t: string) => new Octokit({ auth: t })); + const delay = deps.delay ?? defaultDelay; + const octokit = buildOctokit(token); + + let username: string; + try { + const { data } = await octokit.rest.users.getAuthenticated(); + username = data.login; + } catch (err) { + // 401 here = the stored token is no longer valid (revoked / expired). + // Surface as a re-auth signal so the client can kick off OAuth again. + if (err instanceof Error && 'status' in err && (err as { status: number }).status === 401) { + return jsonError(401, { + error: 'github_auth_required', + message: 'GitHub token was revoked. Please reconnect your account.', + }); + } + return jsonError(502, { + error: 'github_api_error', + message: 'Could not reach GitHub. Try again in a moment.', + details: { stage: 'getAuthenticated' }, + }); + } + + const baseSlug = buildRepoSlug(datasetName); + let repoName: string | null; + try { + repoName = await pickAvailableRepoName(octokit, username, baseSlug); + } catch (err) { + return jsonError(502, { + error: 'github_api_error', + message: 'Could not check repo name availability.', + details: { + stage: 'pickAvailableRepoName', + cause: err instanceof Error ? err.message : String(err), + }, + }); + } + if (!repoName) { + return jsonError(422, { + error: 'github_api_error', + message: + 'All candidate repo names are taken. Try renaming an existing repo on GitHub.', + }); + } + + try { + await octokit.rest.repos.createUsingTemplate({ + template_owner: TEMPLATE_OWNER, + template_repo: TEMPLATE_REPO, + owner: username, + name: repoName, + private: true, + include_all_branches: false, + description: `NDI analysis derived from ${datasetName} on ndi-cloud.com`, + }); + } catch (err) { + const status = + err instanceof Error && 'status' in err + ? (err as { status: number }).status + : 502; + return jsonError(status === 404 ? 502 : status, { + error: + status === 404 + ? 'template_unavailable' + : status === 422 + ? 'github_api_error' + : 'github_api_error', + message: + status === 404 + ? 'The ndi-analysis-template repo is not accessible to GitHub right now.' + : 'Failed to create the new repo from the template.', + details: { + stage: 'createUsingTemplate', + cause: err instanceof Error ? err.message : String(err), + }, + }); + } + + const ready = await pollUntilReady(octokit, username, repoName, delay); + if (!ready) { + // The repo was created but isn't ready yet. We still return success + // with the URL — the user can refresh in a moment. The + // current_analysis.py commit is skipped here; the user can fork in + // their own copy or re-run the action. + return NextResponse.json({ + url: `https://github.com/${username}/${repoName}`, + name: repoName, + owner: username, + note: 'Repo created but not yet ready; current_analysis.py was not committed. Open the URL to retry.', + }); + } + + const analysisFile = generateCurrentAnalysis(panelState, { question }); + try { + await octokit.rest.repos.createOrUpdateFileContents({ + owner: username, + repo: repoName, + path: 'current_analysis.py', + message: 'Initialize current_analysis.py from ndi-cloud.com workspace', + content: Buffer.from(analysisFile, 'utf8').toString('base64'), + }); + } catch (err) { + // Audit 2026-05-20 P1 — sanitize the note field. Pre-fix this + // string interpolated the raw Octokit error message (potentially + // containing GitHub API response bodies that expose the repo's + // internal state). Log the raw error server-side; return a fixed + // user-safe message that doesn't leak upstream details. + logEvent('github.create_repo.commit_failed', { + owner: username, + repo: repoName, + cause: err instanceof Error ? err.message : 'unknown', + }); + // Don't fail the whole request — the repo is live + the user has + // the URL. Note the failure in the response so the UI can warn. + return NextResponse.json({ + url: `https://github.com/${username}/${repoName}`, + name: repoName, + owner: username, + note: + 'Repo created, but the initial `current_analysis.py` commit failed. ' + + 'Open the URL to retry from the GitHub UI — refreshing the page on ' + + 'ndi-cloud.com and re-clicking Open in GitHub will also retry the commit.', + }); + } + + return NextResponse.json({ + url: `https://github.com/${username}/${repoName}`, + name: repoName, + owner: username, + }); +} + +export async function POST(req: Request): Promise { + return handlePost(req); +} diff --git a/apps/web/app/api/github/download-analysis-zip/route.ts b/apps/web/app/api/github/download-analysis-zip/route.ts new file mode 100644 index 00000000..6e1e803c --- /dev/null +++ b/apps/web/app/api/github/download-analysis-zip/route.ts @@ -0,0 +1,280 @@ +/** + * POST /api/github/download-analysis-zip — no-OAuth fallback for users + * who don't want to authenticate against GitHub. Returns a `.zip` + * containing the template repo + an injected `current_analysis.py` + * matching the user's panel args (ADR-010). + * + * Flow: + * 1. Validate env: GITHUB_APP_TOKEN must be set. The template repo + * is private, so we need a server-side PAT to download the + * tarball. Missing → 503 `feature_not_configured`. + * 2. Validate the request body (zod). Bad shape → 400 `invalid_input`. + * 3. Stream the template tarball via + * `octokit.rest.repos.downloadTarballArchive`. + * 4. Unpack the tar entries in-memory (`tar-stream`), inject the + * generated `current_analysis.py`, re-pack as a `.zip` + * (`archiver`). + * 5. Stream the zip back with `Content-Disposition: attachment; + * filename="ndi-.zip"`. + * + * Why we don't shell out to `git clone`: the template is private and + * relatively small (~20 files, <50 KB). Fetching the tarball + repack + * is a single network round-trip + a deterministic in-memory transform. + * Cleaner than provisioning git on Vercel. + */ +import { PassThrough, Readable } from 'node:stream'; +import archiver from 'archiver'; +import extract from 'tar-stream'; +import { createGunzip } from 'node:zlib'; +import { Octokit } from '@octokit/rest'; + +import { generateCurrentAnalysis } from '@/lib/ndi/code-export/current-analysis'; +import { env } from '@/lib/env'; +import { buildRepoSlug } from '@/lib/github/slug'; +import { + GithubAnalysisRequestSchema, + TEMPLATE_OWNER, + TEMPLATE_REPO, + type GithubErrorEnvelope, +} from '@/lib/github/types'; +import { logEvent } from '@/lib/ndi/tools/shared'; + +/** + * Audit 2026-05-20 P0 #2 — gate the route on an NDI session presence. + * + * Pre-fix, the route checked only that `GITHUB_APP_TOKEN` was set and + * the body validated, then used the cloud-app's fine-grained PAT to + * read the PRIVATE `ndi-analysis-template` repo on behalf of any + * unauthenticated visitor. That made the route a free anonymous proxy + * into the template repo (rate-limit-burning + minor data exfiltration). + * + * We don't require the NDI user's full XSRF round-trip (the template + * content is shippable to anyone we've already shipped a session to), + * just the presence of the FastAPI session cookie that we cookie-set + * from /login. That blocks unauthenticated callers cheaply without + * adding a Railway round-trip on every download. + */ +const SESSION_COOKIE_NAMES = ['session', 'ndi-session']; + +function hasNdiSession(req: Request): boolean { + const cookie = req.headers.get('cookie'); + if (!cookie) return false; + // Cookie parsing: split on `;`, trim, look for one of our session + // names. We don't validate the contents — the route doesn't talk to + // FastAPI; it just needs to know the caller has been issued a session + // by /login. A fake cookie value is no worse than the previous open + // state because the route exposes no per-user data. + const parts = cookie.split(';').map((p) => p.trim()); + for (const part of parts) { + const eq = part.indexOf('='); + if (eq <= 0) continue; + const name = part.slice(0, eq); + if (SESSION_COOKIE_NAMES.includes(name)) return true; + } + return false; +} + +// Tarball size cap. The template ships <50 KB today and shouldn't grow +// past a small multiple of that; if a future template includes test +// fixtures or notebooks, the operator can bump this with intent. A +// runaway tarball would otherwise sit in heap before streaming begins. +const MAX_TARBALL_BYTES = 5_000_000; + +export const runtime = 'nodejs'; +export const maxDuration = 60; + +export interface DownloadZipDeps { + /** Inject an Octokit factory for tests. Defaults to the real constructor. */ + buildOctokit?: (token: string) => Octokit; +} + +function jsonError(status: number, body: GithubErrorEnvelope): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); +} + +/** + * Internal handler exported for tests. Returns either a JSON error + * response or a streaming zip response. + */ +export async function handlePost( + req: Request, + deps: DownloadZipDeps = {}, +): Promise { + // Audit 2026-05-20 P0 #2 — refuse anonymous calls. Done BEFORE env + // checks + body parse so we can't be probed for env-presence by an + // unauthenticated visitor. + if (!hasNdiSession(req)) { + logEvent('github.download_zip.no_session'); + return jsonError(401, { + error: 'invalid_input', + message: 'You must be signed in to download an analysis template.', + }); + } + + const appToken = env.GITHUB_APP_TOKEN; + if (!appToken) { + return jsonError(503, { + error: 'feature_not_configured', + message: + 'ZIP download is not configured. Contact ops to enable the GitHub integration.', + }); + } + + let body: unknown; + try { + body = await req.json(); + } catch { + return jsonError(400, { + error: 'invalid_input', + message: 'Request body must be valid JSON.', + }); + } + + const parsed = GithubAnalysisRequestSchema.safeParse(body); + if (!parsed.success) { + return jsonError(400, { + error: 'invalid_input', + message: 'Request body failed validation.', + details: { issues: parsed.error.issues }, + }); + } + + const { panelState, datasetName, question } = parsed.data; + + const buildOctokit = + deps.buildOctokit ?? ((t: string) => new Octokit({ auth: t })); + const octokit = buildOctokit(appToken); + + // 1. Download the template tarball. `downloadTarballArchive` returns + // a 302 to a short-lived S3 URL; @octokit/request follows it. + let tarBuffer: Buffer; + try { + const tarResp = await octokit.rest.repos.downloadTarballArchive({ + owner: TEMPLATE_OWNER, + repo: TEMPLATE_REPO, + ref: 'main', + }); + // octokit returns `data: ArrayBuffer` for the tarball. + tarBuffer = Buffer.from(tarResp.data as ArrayBuffer); + // Audit 2026-05-20 P2 — explicit size ceiling. The template is + // ~50 KB today; if it grows past MAX_TARBALL_BYTES we fail fast + // rather than buffer arbitrary payloads into Vercel function heap. + if (tarBuffer.byteLength > MAX_TARBALL_BYTES) { + logEvent('github.download_zip.tarball_too_large', { + bytes: tarBuffer.byteLength, + cap: MAX_TARBALL_BYTES, + }); + return jsonError(413, { + error: 'template_unavailable', + message: 'Template archive exceeds the supported size limit.', + details: { bytes: tarBuffer.byteLength, cap: MAX_TARBALL_BYTES }, + }); + } + } catch (err) { + return jsonError(502, { + error: 'template_unavailable', + message: 'Could not fetch the analysis template.', + details: { + stage: 'downloadTarballArchive', + cause: err instanceof Error ? err.message : String(err), + }, + }); + } + + // 2. Build the zip stream. We pipe through a PassThrough so the + // response Body can read it as a web-stream. + const zip = archiver('zip', { zlib: { level: 6 } }); + const out = new PassThrough(); + zip.pipe(out); + + const analysisFile = generateCurrentAnalysis(panelState, { question }); + const slug = buildRepoSlug(datasetName); + + // 3. Untar in-memory; for each entry, push to the zip (renaming + // the top-level directory from GitHub's + // `Waltham-Data-Science-ndi-analysis-template-` to our + // slug). Inject our current_analysis.py last so it overrides + // any same-named template file. + const extractStream = extract.extract(); + const transformPromise = new Promise((resolve, reject) => { + extractStream.on('entry', (header, stream, next) => { + // Skip the top-level directory entry itself. + if (header.type !== 'file') { + stream.resume(); + stream.on('end', next); + return; + } + // Strip the prefix dir; replace with our slug. + const segments = header.name.split('/'); + segments.shift(); // drop GitHub's auto-generated top dir + const newPath = `${slug}/${segments.join('/')}`; + + // If the template happens to ship a current_analysis.py.example + // or similar, keep it — we only INJECT a new file, never strip. + const chunks: Buffer[] = []; + stream.on('data', (chunk: Buffer) => chunks.push(chunk)); + stream.on('end', () => { + zip.append(Buffer.concat(chunks), { name: newPath }); + next(); + }); + stream.on('error', (err) => reject(err)); + }); + extractStream.on('finish', () => { + // Inject the generated current_analysis.py at the repo root. + zip.append(analysisFile, { name: `${slug}/current_analysis.py` }); + zip.finalize().catch(reject); + resolve(); + }); + extractStream.on('error', reject); + }); + + // Feed the gzipped tarball through gunzip → extract. + const tarReadable = Readable.from(tarBuffer); + tarReadable.pipe(createGunzip()).pipe(extractStream); + + // Wait for the extract → zip transform to finish setting up; the + // actual write to the response body streams from `out` into the + // Response immediately so the user starts downloading right away. + try { + await transformPromise; + } catch (err) { + return jsonError(500, { + error: 'github_api_error', + message: 'Failed to repack the template into a zip.', + details: { cause: err instanceof Error ? err.message : String(err) }, + }); + } + + // PassThrough is a Node Readable; Web Response wants a web ReadableStream. + const webStream = streamFromPassThrough(out); + return new Response(webStream, { + status: 200, + headers: { + 'content-type': 'application/zip', + 'content-disposition': `attachment; filename="${slug}.zip"`, + 'cache-control': 'no-store', + }, + }); +} + +function streamFromPassThrough(pt: PassThrough): ReadableStream { + return new ReadableStream({ + start(controller) { + pt.on('data', (chunk: Buffer) => + controller.enqueue(new Uint8Array(chunk)), + ); + pt.on('end', () => controller.close()); + pt.on('error', (err) => controller.error(err)); + }, + cancel() { + pt.destroy(); + }, + }); +} + +export async function POST(req: Request): Promise { + return handlePost(req); +} diff --git a/apps/web/app/api/github/oauth/callback/route.ts b/apps/web/app/api/github/oauth/callback/route.ts new file mode 100644 index 00000000..9bec7f89 --- /dev/null +++ b/apps/web/app/api/github/oauth/callback/route.ts @@ -0,0 +1,163 @@ +/** + * GET /api/github/oauth/callback — completes the GitHub OAuth dance. + * + * Verifies the CSRF state nonce matches the cookie set at + * /api/github/oauth/start, exchanges the code for an access token, + * stores the token (encrypted) + username in cookies, and redirects + * to the returnTo path stashed in a sibling cookie. + * + * On any verification / exchange failure, returns a JSON error to + * help the user debug. Production wires the button to retry the OAuth + * flow on its next click. + */ +import { NextResponse } from 'next/server'; + +import { env } from '@/lib/env'; +import { + buildLinkCookies, + exchangeOAuthCode, + readCookie, +} from '@/lib/github/oauth'; + +export const runtime = 'nodejs'; + +const STATE_COOKIE = 'ndi-gh-oauth-state'; +const RETURN_TO_COOKIE = 'ndi-gh-oauth-return-to'; + +/** + * Audit 2026-05-20 P1 — gatekeeper for the post-OAuth redirect target. + * + * Rejects everything that isn't an unambiguously same-origin path: + * + * - must START with `/` + * - must NOT start with `//` (protocol-relative URLs like + * `//evil.com/foo` resolve to the attacker's domain) + * - must NOT contain whitespace, control chars, or a backslash + * (defends against URL-parser quirks across browsers) + * - must NOT contain a scheme separator anywhere in the path + * + * The matching helper at `/api/github/oauth/start` also gates the + * value before writing the cookie; this is the second line of defense + * — a hostile cookie injection (subdomain takeover, MITM with stale + * cert, etc.) can't pivot the callback into an open redirect. + */ +function isSafeReturnPath(value: string): boolean { + if (typeof value !== 'string') return false; + if (value.length === 0 || value.length > 512) return false; + if (!value.startsWith('/')) return false; + if (value.startsWith('//')) return false; + // Forbid backslashes (legacy IE / Edge would resolve `/\evil.com` as + // `//evil.com`). Forbid whitespace and ASCII control chars (0x00-0x1F + // and 0x7F DEL) — non-range entries so the engine can't accidentally + // widen the character class. + for (let i = 0; i < value.length; i += 1) { + const code = value.charCodeAt(i); + if (code < 0x20) return false; // control chars + if (code === 0x7f) return false; // DEL + if (code === 0x5c) return false; // backslash + if (code === 0x20) return false; // space + } + // Disallow scheme markers anywhere — `/foo:javascript:` could be + // coerced into a javascript URL on some legacy paths. + if (/^\/[a-zA-Z][a-zA-Z0-9+.-]*:/.test(value)) return false; + return true; +} + +function clearTransientCookies(res: NextResponse): void { + const secure = process.env.NODE_ENV !== 'test'; + res.headers.append( + 'Set-Cookie', + `${STATE_COOKIE}=; Path=/api/github/oauth; HttpOnly; SameSite=Lax; Max-Age=0${secure ? '; Secure' : ''}`, + ); + res.headers.append( + 'Set-Cookie', + `${RETURN_TO_COOKIE}=; Path=/api/github/oauth; HttpOnly; SameSite=Lax; Max-Age=0${secure ? '; Secure' : ''}`, + ); +} + +export async function GET(req: Request): Promise { + const clientId = env.GITHUB_CLIENT_ID; + const clientSecret = env.GITHUB_CLIENT_SECRET; + if (!clientId || !clientSecret) { + return NextResponse.json( + { + error: 'feature_not_configured', + message: 'GitHub integration is not configured.', + }, + { status: 503 }, + ); + } + + const url = new URL(req.url); + const code = url.searchParams.get('code'); + const state = url.searchParams.get('state'); + if (!code || !state) { + const res = NextResponse.json( + { + error: 'invalid_input', + message: 'Missing code or state.', + }, + { status: 400 }, + ); + clearTransientCookies(res); + return res; + } + + const cookieHeader = req.headers.get('cookie'); + const expectedState = readCookie(cookieHeader, STATE_COOKIE); + if (!expectedState || expectedState !== state) { + const res = NextResponse.json( + { + error: 'invalid_input', + message: 'OAuth state mismatch — possible CSRF. Restart the flow.', + }, + { status: 400 }, + ); + clearTransientCookies(res); + return res; + } + + let token: string; + let username: string; + try { + const exchanged = await exchangeOAuthCode({ + clientId, + clientSecret, + code, + redirectUri: `${url.origin}/api/github/oauth/callback`, + }); + token = exchanged.token; + username = exchanged.username; + } catch (err) { + const res = NextResponse.json( + { + error: 'github_api_error', + message: + 'Failed to exchange the OAuth code with GitHub. Please retry.', + details: { cause: err instanceof Error ? err.message : String(err) }, + }, + { status: 502 }, + ); + clearTransientCookies(res); + return res; + } + + // Audit 2026-05-20 P1 — `returnTo` is read from a sibling cookie + // (`ndi-gh-oauth-return-to`) set at /oauth/start. The previous code + // wrapped it in `new URL(returnTo, origin)` which constrains + // *relative* values to same-origin BUT silently lets an absolute + // URL override the base → open redirect via cookie injection. Lock + // the value to a path-only shape via isSafeReturnPath() above. + const rawReturnTo = + decodeURIComponent(readCookie(cookieHeader, RETURN_TO_COOKIE) ?? '') || '/'; + const returnTo = isSafeReturnPath(rawReturnTo) ? rawReturnTo : '/'; + + const res = NextResponse.redirect(new URL(returnTo, url.origin), { + status: 302, + }); + clearTransientCookies(res); + for (const cookie of buildLinkCookies(token, username)) { + res.headers.append('Set-Cookie', cookie); + } + return res; +} diff --git a/apps/web/app/api/github/oauth/start/route.ts b/apps/web/app/api/github/oauth/start/route.ts new file mode 100644 index 00000000..59a018a6 --- /dev/null +++ b/apps/web/app/api/github/oauth/start/route.ts @@ -0,0 +1,76 @@ +/** + * GET /api/github/oauth/start — kicks off the GitHub OAuth dance. + * + * Generates a CSRF `state` nonce, stashes it in a short-lived + * HttpOnly cookie, and redirects the browser to GitHub's authorize + * URL. The callback at `/api/github/oauth/callback` verifies the + * state, exchanges the code for a token, and persists the token in + * the `ndi-gh-token` cookie. + * + * Query params: + * - `returnTo` — where to send the browser after the callback + * completes. Constrained to same-origin paths to prevent open-redirect. + * + * If the env vars aren't configured, returns 503 — the button is + * gated client-side via `NEXT_PUBLIC_GITHUB_INTEGRATION_ENABLED`, so + * this is mostly defense-in-depth for direct route hits. + */ +import { randomBytes } from 'node:crypto'; +import { NextResponse } from 'next/server'; + +import { env } from '@/lib/env'; +import { buildAuthorizeUrl } from '@/lib/github/oauth'; + +export const runtime = 'nodejs'; + +const STATE_COOKIE = 'ndi-gh-oauth-state'; +const RETURN_TO_COOKIE = 'ndi-gh-oauth-return-to'; +const STATE_MAX_AGE_SECONDS = 600; // 10 min + +function isSafeReturnPath(input: string | null): string { + if (!input) return '/'; + // Reject anything that looks like a host (`//foo.com`) or a full URL. + if (input.startsWith('//') || input.includes('://')) return '/'; + // Must start with `/` to be a path, never a relative URL. + if (!input.startsWith('/')) return '/'; + return input; +} + +export async function GET(req: Request): Promise { + const clientId = env.GITHUB_CLIENT_ID; + const clientSecret = env.GITHUB_CLIENT_SECRET; + if (!clientId || !clientSecret) { + return NextResponse.json( + { + error: 'feature_not_configured', + message: 'GitHub integration is not configured.', + }, + { status: 503 }, + ); + } + + const url = new URL(req.url); + const returnTo = isSafeReturnPath(url.searchParams.get('returnTo')); + const state = randomBytes(24).toString('hex'); + + // Build the absolute redirect URI: the OAuth callback on the + // current origin (matches what the user registered on GitHub). + const redirectUri = `${url.origin}/api/github/oauth/callback`; + const authorizeUrl = buildAuthorizeUrl({ + clientId, + redirectUri, + state, + }); + + const res = NextResponse.redirect(authorizeUrl, { status: 302 }); + const secure = process.env.NODE_ENV !== 'test'; + res.headers.append( + 'Set-Cookie', + `${STATE_COOKIE}=${state}; Path=/api/github/oauth; HttpOnly; SameSite=Lax; Max-Age=${STATE_MAX_AGE_SECONDS}${secure ? '; Secure' : ''}`, + ); + res.headers.append( + 'Set-Cookie', + `${RETURN_TO_COOKIE}=${encodeURIComponent(returnTo)}; Path=/api/github/oauth; HttpOnly; SameSite=Lax; Max-Age=${STATE_MAX_AGE_SECONDS}${secure ? '; Secure' : ''}`, + ); + return res; +} diff --git a/apps/web/app/api/github/oauth/unlink/route.ts b/apps/web/app/api/github/oauth/unlink/route.ts new file mode 100644 index 00000000..0ba0014a --- /dev/null +++ b/apps/web/app/api/github/oauth/unlink/route.ts @@ -0,0 +1,60 @@ +/** + * POST /api/github/oauth/unlink — clears the local GitHub OAuth + * cookie. Doesn't revoke the token on GitHub's side (that requires + * the user to visit github.com/settings/applications); we just stop + * using it here. + * + * Audit 2026-05-20 P1 — adds an Origin header REQUIREMENT (the + * proxy.ts middleware enforces an allowlist when Origin is present + * but admits requests with NO Origin header at all; this route is a + * cookie-clear and should refuse to operate unless the call is + * unambiguously same-origin from our own browser surface). Belt-and- + * suspenders alongside the proxy.ts P1 tightening. + */ +import { NextResponse } from 'next/server'; + +import { buildUnlinkCookies } from '@/lib/github/oauth'; + +export const runtime = 'nodejs'; + +const ALLOWED_ORIGIN_SUFFIXES = [ + 'https://ndi-cloud.com', + 'https://www.ndi-cloud.com', +]; + +function isSameOriginRequest(req: Request): boolean { + const origin = req.headers.get('origin'); + if (!origin) return false; + if (ALLOWED_ORIGIN_SUFFIXES.includes(origin)) return true; + // Preview Vercel URLs (`*.vercel.app`) — accept the per-deployment + // domain at request time. We don't pin a specific preview host because + // the preview URL changes per branch. + try { + const u = new URL(origin); + if (u.hostname.endsWith('.vercel.app')) return true; + // Local dev (`http://localhost:3000`). + if ( + process.env.NODE_ENV !== 'production' && + (u.hostname === 'localhost' || u.hostname === '127.0.0.1') + ) { + return true; + } + } catch { + return false; + } + return false; +} + +export async function POST(req: Request): Promise { + if (!isSameOriginRequest(req)) { + return NextResponse.json( + { error: 'origin_required', message: 'Cross-origin or origin-less unlink is not allowed.' }, + { status: 403 }, + ); + } + const res = NextResponse.json({ ok: true }); + for (const cookie of buildUnlinkCookies()) { + res.headers.append('Set-Cookie', cookie); + } + return res; +} diff --git a/apps/web/app/api/github/status/route.ts b/apps/web/app/api/github/status/route.ts new file mode 100644 index 00000000..3ba24f10 --- /dev/null +++ b/apps/web/app/api/github/status/route.ts @@ -0,0 +1,40 @@ +/** + * GET /api/github/status — quick check of whether the cloud-app has a + * GitHub OAuth token for this browser, and what username it's linked + * to. + * + * Reads the non-HttpOnly `ndi-gh-user` cookie set by the OAuth + * callback. Doesn't decrypt the token — that's intentional, this + * route never touches the encryption key, just confirms presence. + * + * Also surfaces server-side feature configuration so the client + * doesn't have to look at `NEXT_PUBLIC_GITHUB_INTEGRATION_ENABLED` — + * it can rely on the merged verdict. + */ +import { NextResponse } from 'next/server'; + +import { env } from '@/lib/env'; +import { + GITHUB_TOKEN_COOKIE, + GITHUB_USER_COOKIE, + readCookie, +} from '@/lib/github/oauth'; + +export const runtime = 'nodejs'; + +export async function GET(req: Request): Promise { + const featureConfigured = Boolean( + env.GITHUB_CLIENT_ID && env.GITHUB_CLIENT_SECRET, + ); + const downloadConfigured = Boolean(env.GITHUB_APP_TOKEN); + const cookie = req.headers.get('cookie'); + const hasToken = Boolean(readCookie(cookie, GITHUB_TOKEN_COOKIE)); + const username = readCookie(cookie, GITHUB_USER_COOKIE); + + return NextResponse.json({ + featureConfigured, + downloadConfigured, + linked: hasToken, + username: hasToken ? username : null, + }); +} diff --git a/apps/web/components/ai/AskHeroQuickInput.tsx b/apps/web/components/ai/AskHeroQuickInput.tsx new file mode 100644 index 00000000..62a91e06 --- /dev/null +++ b/apps/web/components/ai/AskHeroQuickInput.tsx @@ -0,0 +1,124 @@ +'use client'; + +/** + * AskHeroQuickInput — compact inline input intended to drop into the + * workspace hero band. + * + * Phase D of the workspace redesign. Two affordances: + * + * 1. Pressing `/` from anywhere in the workspace (when no input is + * focused) focuses this input. Matches the Linear / Notion + * search-bar pattern. + * 2. Submitting the input opens the Ask panel in drawer mode. + * + * Phase D limitation: the "pre-send on open" wiring requires AskShell + * to accept an `initialInput` / `sendOnMount` mechanism, which in turn + * needs a shared ephemeral store (Zustand atom or a React context) + * that AskShell drains on first mount. Implementing that store is + * deferred to a Phase E follow-up so it doesn't block the Phase D + * merge. Current behavior: submitting opens the panel — the typed + * text appears in the panel input field instead of being pre-sent. + * Still a useful flow; just one extra Enter press. + * + * White-on-dark theming so the input reads on top of the depth + * gradient in the workspace hero. The hint chip on the right shows + * `/` for the focus shortcut. + */ +import { Send } from 'lucide-react'; +import { useCallback, useEffect, useRef, useState } from 'react'; + +import { useAskPanelState } from '@/lib/ai/use-ask-panel-state'; + +interface AskHeroQuickInputProps { + /** Placeholder text. Defaults to "Ask about this dataset…" */ + placeholder?: string; + className?: string; +} + +export function AskHeroQuickInput({ + placeholder = 'Ask about this dataset…', + className, +}: AskHeroQuickInputProps) { + const [value, setValue] = useState(''); + const { openPanel } = useAskPanelState(); + const inputRef = useRef(null); + + // `/` from anywhere in the workspace focuses this input. Focus + // guard: skip if the user is already typing in an input/textarea + // (don't steal the "/" key from a filter). + const handleGlobalKeyDown = useCallback((e: KeyboardEvent) => { + const target = e.target as HTMLElement; + const isInput = + target.tagName === 'INPUT' || + target.tagName === 'TEXTAREA' || + target.tagName === 'SELECT' || + target.isContentEditable; + + if (e.key === '/' && !isInput && !e.metaKey && !e.ctrlKey) { + e.preventDefault(); + inputRef.current?.focus(); + } + }, []); + + useEffect(() => { + document.addEventListener('keydown', handleGlobalKeyDown); + return () => document.removeEventListener('keydown', handleGlobalKeyDown); + }, [handleGlobalKeyDown]); + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + // Open the panel — whether or not the user typed anything. An + // empty submit still opens the panel (matches Linear's behavior). + // TODO (Phase E): if value is non-empty, write to a pending-send + // store and have AskShell drain it on mount. + openPanel(); + setValue(''); + }; + + return ( + +
+ setValue(e.target.value)} + placeholder={placeholder} + aria-label={placeholder} + className={[ + 'w-full rounded-lg px-3.5 py-2 text-[13.5px] leading-tight', + 'bg-white/15 border border-white/25 text-white placeholder:text-white/50', + 'focus:outline-none focus:bg-white/20 focus:border-white/40', + 'transition-colors duration-(--duration-base) ease-(--ease-out)', + 'pr-10', + ].join(' ')} + /> + + / + +
+ + + ); +} diff --git a/apps/web/components/ai/AskKeyboardShortcuts.tsx b/apps/web/components/ai/AskKeyboardShortcuts.tsx new file mode 100644 index 00000000..80b49ee8 --- /dev/null +++ b/apps/web/components/ai/AskKeyboardShortcuts.tsx @@ -0,0 +1,74 @@ +'use client'; + +/** + * AskKeyboardShortcuts — global keyboard handler for the workspace + * Ask panel. + * + * Phase D of the workspace redesign. Renders nothing — it is a pure + * `useEffect` mount that registers and cleans up document-level + * listeners. Drop it once in the workspace layout tree. + * + * Registered shortcuts: + * - Cmd+K / Ctrl+K → open panel (no-op when already open) + * - Cmd+\ / Ctrl+\ → cycle modes forward (drawer → sidebar → fullscreen) + * - / → focus AskHeroQuickInput (handled by that + * component; documented here for completeness) + * - Esc → close panel (AskPanel itself handles this; + * listed here for completeness) + * + * Focus guard: all shortcuts skip when the focused element is INPUT, + * TEXTAREA, SELECT, or contenteditable. This component does NOT + * register an Esc listener — AskPanel owns that — because a global + * Esc would also fire when the user is just trying to blur a + * workspace filter input. + * + * Co-existence: the Cmd+K listener here is redundant with + * AskPanelTrigger's own Cmd+K listener. Both calling `openPanel()` + * is safe because `openPanel` is a no-op when the panel is already + * open. We keep both so neither component depends on the other for + * the shortcut to work. + */ +import { useCallback, useEffect } from 'react'; + +import { useAskPanelState } from '@/lib/ai/use-ask-panel-state'; + +export function AskKeyboardShortcuts() { + const { openPanel, expand } = useAskPanelState(); + + const handleKeyDown = useCallback( + (e: KeyboardEvent) => { + const target = e.target as HTMLElement; + const isInput = + target.tagName === 'INPUT' || + target.tagName === 'TEXTAREA' || + target.tagName === 'SELECT' || + target.isContentEditable; + + if (isInput) return; + + const meta = e.metaKey || e.ctrlKey; + + // Cmd+K → open. No-op when open; redundant with AskPanelTrigger. + if (meta && e.key === 'k') { + e.preventDefault(); + openPanel(); + return; + } + + // Cmd+\ → cycle modes forward. + if (meta && e.key === '\\') { + e.preventDefault(); + expand(); + return; + } + }, + [openPanel, expand], + ); + + useEffect(() => { + document.addEventListener('keydown', handleKeyDown); + return () => document.removeEventListener('keydown', handleKeyDown); + }, [handleKeyDown]); + + return null; +} diff --git a/apps/web/components/ai/AskPanel.tsx b/apps/web/components/ai/AskPanel.tsx new file mode 100644 index 00000000..6d912312 --- /dev/null +++ b/apps/web/components/ai/AskPanel.tsx @@ -0,0 +1,570 @@ +'use client'; + +/** + * AskPanel — the three-mode workspace chat panel. + * + * Phase D of the workspace redesign (2026-05-16). Renders AskShell + * inside a panel chrome that supports three expansion modes the user + * cycles between: + * + * Drawer (default): + * 420px right-side overlay, slides in from right, white surface, + * shadow-xl. Overlays workspace content. Dismissable with Esc + + * close button. Does NOT have a click-outside dismiss to avoid + * losing a conversation mid-sentence. + * + * Sidebar: + * 520px right-side persistent column. No overlay backdrop. The + * panel renders at its full width and the parent layout is + * responsible for reflowing workspace content (`data-ask-panel-mode` + * attribute on the panel + a CSS rule on the layout would do it). + * For Phase D v1 the sidebar overlays — Phase E adds the layout + * reflow. + * + * Fullscreen: + * Takes the full viewport. Workspace stays in URL but is visually + * hidden behind the panel. Chat log centered, max-w-[760px], + * matching ChatGPT / Claude.ai layout. + * + * Mode controls (toolbar buttons in the header): + * ⤢ Expand — cycles drawer → sidebar → fullscreen (stops at max) + * ⤡ Contract — cycles fullscreen → sidebar → drawer (stops at min) + * × Close — removes ?ask from the URL + * Esc — same as Close (handled globally via useEffect) + * + * ARIA: `role="dialog"` + `aria-modal="true"` for drawer and + * fullscreen (they overlay content). Sidebar is `role="complementary"` + * (persistent, not modal). The close button gets initial focus when + * the panel opens so keyboard users land inside the dialog. + * + * Renders null when `?ask` is absent — no DOM at all. + */ +import { Maximize2, MessageSquare, Minimize2, X } from 'lucide-react'; +import type { RefObject } from 'react'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; + +import { AskShell, type AskShellContext } from '@/components/ai/AskShell'; +import { cn } from '@/lib/cn'; +import { + subscribeToAskPrefill, + type AskPrefillPayload, +} from '@/lib/ai/ask-prefill-bus'; +import { useAskPanelState } from '@/lib/ai/use-ask-panel-state'; +import { useWorkspaceSelection } from '@/lib/workspace/use-workspace-selection'; + +export interface AskPanelProps { + /** + * Baseline context from the workspace layout (datasetId, + * datasetName). AskPanel enriches it with live selection state + * read from `useWorkspaceSelection` — when the user picks a + * subject/session/etc., subsequent chat turns carry that selection + * automatically. + * + * Phase F (W7 audit fix). Pre-fix, context was theatre only; the + * AskPanel header read "Asking about: <dataset>" with zero + * API impact. Post-fix, the selection IS forwarded to /api/ask. + */ + context?: AskShellContext; +} + +export function AskPanel({ context }: AskPanelProps) { + const { open, mode, openPanel, expand, contract, close } = useAskPanelState(); + const { selection } = useWorkspaceSelection(); + + // Phase G — listen for "Ask Claude about these" gestures from + // anywhere in the workspace (today: WorkspaceDataGrid bulk-actions + // bar). On event: open the panel (if closed) and forward the + // payload to AskShell, which stages text + optionally auto-sends. + // The staged value clears after consumption so re-renders don't + // double-fire. + const [pendingPrefill, setPendingPrefill] = + useState(null); + useEffect(() => { + const unsubscribe = subscribeToAskPrefill((payload) => { + setPendingPrefill(payload); + openPanel(); + }); + return unsubscribe; + }, [openPanel]); + const handlePrefillConsumed = useCallback(() => { + setPendingPrefill(null); + }, []); + + // Merge selection into the baseline context. AskShell stringifies + // this to detect transport rebuilds, so we don't include null / + // undefined keys — they'd flap the JSON stable-ish. + const enrichedContext: AskShellContext | undefined = useMemo(() => { + const base: AskShellContext = { ...context }; + if (selection.subject) base.selectedSubjectId = selection.subject; + if (selection.session) base.selectedSessionId = selection.session; + if (selection.probe) base.selectedProbeId = selection.probe; + if (selection.stimulus) base.selectedStimulusId = selection.stimulus; + if (selection.unit) base.selectedUnitId = selection.unit; + return Object.keys(base).length > 0 ? base : undefined; + }, [ + context, + selection.subject, + selection.session, + selection.probe, + selection.stimulus, + selection.unit, + ]); + + // Focus close button when the panel opens — keyboard users should + // land inside the dialog, not behind it. + const closeButtonRef = useRef(null); + useEffect(() => { + if (open) { + const t = setTimeout(() => closeButtonRef.current?.focus(), 50); + return () => clearTimeout(t); + } + return undefined; + }, [open]); + + // Esc closes the panel from anywhere inside it. + useEffect(() => { + if (!open) return; + const onKey = (e: KeyboardEvent) => { + if (e.key === 'Escape') { + e.stopPropagation(); + close(); + } + }; + document.addEventListener('keydown', onKey, true); + return () => document.removeEventListener('keydown', onKey, true); + }, [open, close]); + + if (!open) return null; + + const canExpand = mode !== 'fullscreen'; + const canContract = mode !== 'drawer'; + + const title = 'Ask'; + const contextLine = context?.datasetName + ? `Asking about: ${context.datasetName}` + : null; + + if (mode === 'fullscreen') { + return ( + + ); + } + + if (mode === 'sidebar') { + return ( + + ); + } + + // Default: drawer + return ( + + ); +} + +/* -------------------------------------------------------------------------- */ +/* Shared header toolbar */ +/* -------------------------------------------------------------------------- */ + +interface PanelHeaderProps { + title: string; + contextLine: string | null; + canExpand: boolean; + canContract: boolean; + onExpand?: () => void; + onContract?: () => void; + onClose: () => void; + closeButtonRef: RefObject; +} + +function PanelHeader({ + title, + contextLine, + canExpand, + canContract, + onExpand, + onContract, + onClose, + closeButtonRef, +}: PanelHeaderProps) { + return ( +
+ {/* Audit 2026-05-18 (G-verify B1): the left title block was + `flex min-w-0` but WITHOUT `flex-1`. Without explicit grow, + flex-basis defaulted to content width — a long dataset + title pushed the toolbar buttons off-screen on the 419px + drawer (verified: header scrollWidth=940 but client=419, + close X at x=1752). Adding `flex-1` lets the title block + claim the remaining row width; the inner `min-w-0` + + `truncate` chain then engages correctly. */} +
+ +
+

+ {title} +

+ {contextLine && ( +

+ {contextLine} +

+ )} +
+
+ +
+ {onExpand && ( + + + + )} + {onContract && ( + + + + )} + + Esc + + +
+
+ ); +} + +function ToolbarButton({ + children, + disabled, + onClick, + 'aria-label': ariaLabel, + title, +}: { + children: React.ReactNode; + disabled?: boolean; + onClick: () => void; + 'aria-label': string; + title?: string; +}) { + return ( + + ); +} + +/* -------------------------------------------------------------------------- */ +/* DrawerPanel */ +/* -------------------------------------------------------------------------- */ + +interface DrawerPanelProps { + title: string; + contextLine: string | null; + context?: AskShellContext; + canExpand: boolean; + onExpand: () => void; + onClose: () => void; + closeButtonRef: RefObject; + prefill: AskPrefillPayload | null; + onPrefillConsumed: () => void; +} + +function DrawerPanel({ + title, + contextLine, + context, + canExpand, + onExpand, + onClose, + closeButtonRef, + prefill, + onPrefillConsumed, +}: DrawerPanelProps) { + return ( + <> + {/* Inert backdrop — visual depth only, no dismiss-on-click. */} +
+
+ + {/* Grid 1fr row — gives the chat a deterministic height for + ChatThread's overflow-y-auto to scroll against. `min-h-0` + prevents grid implicit-min from stretching with content. */} +
+ +
+
+ + + ); +} + +/* -------------------------------------------------------------------------- */ +/* SidebarPanel */ +/* -------------------------------------------------------------------------- */ + +interface SidebarPanelProps { + title: string; + contextLine: string | null; + context?: AskShellContext; + canExpand: boolean; + canContract: boolean; + onExpand: () => void; + onContract: () => void; + onClose: () => void; + closeButtonRef: RefObject; + prefill: AskPrefillPayload | null; + onPrefillConsumed: () => void; +} + +function SidebarPanel({ + title, + contextLine, + context, + canExpand, + canContract, + onExpand, + onContract, + onClose, + closeButtonRef, + prefill, + onPrefillConsumed, +}: SidebarPanelProps) { + // Sidebar: not a modal overlay — `role="complementary"`. v1 still + // renders position:fixed (same as drawer) so it doesn't require + // reflowing the workspace layout. Phase E adds the reflow via a + // sibling-flex layout + data-attribute. + return ( + + ); +} + +/* -------------------------------------------------------------------------- */ +/* FullscreenPanel */ +/* -------------------------------------------------------------------------- */ + +interface FullscreenPanelProps { + title: string; + contextLine: string | null; + context?: AskShellContext; + canContract: boolean; + onContract: () => void; + onClose: () => void; + closeButtonRef: RefObject; + prefill: AskPrefillPayload | null; + onPrefillConsumed: () => void; +} + +function FullscreenPanel({ + title, + contextLine, + context, + canContract, + onContract, + onClose, + closeButtonRef, + prefill, + onPrefillConsumed, +}: FullscreenPanelProps) { + return ( +
+ {/* Fullscreen header — wider, max-width matches workspace shell. */} +
+
+ +
+

+ {contextLine ? `${title} — ${contextLine}` : title} +

+
+
+
+ + + + + Esc + + +
+
+ + {/* Chat area — centered, max-w-[760px] like ChatGPT / Claude.ai. + `min-h-0` propagates the grid's 1fr row height through the + centering wrapper so ChatThread can scroll. */} +
+
+ +
+
+
+ ); +} diff --git a/apps/web/components/ai/AskPanelTrigger.tsx b/apps/web/components/ai/AskPanelTrigger.tsx new file mode 100644 index 00000000..aeccd81e --- /dev/null +++ b/apps/web/components/ai/AskPanelTrigger.tsx @@ -0,0 +1,75 @@ +'use client'; + +/** + * AskPanelTrigger — floating bottom-right button that opens the Ask + * panel. + * + * Phase D of the workspace redesign. Two responsibilities: + * 1. Click → `state.openPanel()`. + * 2. Cmd+K / Ctrl+K → `state.openPanel()`. + * + * Hidden when the panel is already open (no double affordance — the + * panel itself has a close button). + * + * Fixed at bottom-right, z-40 (below the panel at z-50, above tab + * content). 48×48 rounded-full, white surface, brand-blue icon, + * shadow-lg, hover lift. Keyboard hint "K" surfaces via the `title` + * attribute on hover. + * + * Focus guard: the Cmd+K listener skips when the focused element is + * an INPUT, TEXTAREA, SELECT, or contenteditable. Inputs handle the + * shortcut themselves if needed (most don't bind Cmd+K). + */ +import { Sparkles } from 'lucide-react'; +import { useCallback, useEffect } from 'react'; + +import { useAskPanelState } from '@/lib/ai/use-ask-panel-state'; + +export function AskPanelTrigger() { + const { open, openPanel } = useAskPanelState(); + + const handleKeyDown = useCallback( + (e: KeyboardEvent) => { + const target = e.target as HTMLElement; + const isInput = + target.tagName === 'INPUT' || + target.tagName === 'TEXTAREA' || + target.tagName === 'SELECT' || + target.isContentEditable; + + if ((e.metaKey || e.ctrlKey) && e.key === 'k' && !isInput) { + e.preventDefault(); + openPanel(); + } + }, + [openPanel], + ); + + useEffect(() => { + document.addEventListener('keydown', handleKeyDown); + return () => document.removeEventListener('keydown', handleKeyDown); + }, [handleKeyDown]); + + if (open) return null; + + return ( + + ); +} diff --git a/apps/web/components/ai/AskShell.tsx b/apps/web/components/ai/AskShell.tsx new file mode 100644 index 00000000..449742c2 --- /dev/null +++ b/apps/web/components/ai/AskShell.tsx @@ -0,0 +1,541 @@ +'use client'; + +/** + * AskShell — the chat surface reused across all entry points. + * + * Previously lived at `app/(marketing)/ask/ask-shell.tsx`. Moved to + * `components/ai/` in Phase D of the workspace redesign (2026-05-16) + * so it can be imported by `AskPanel` without a cross-route-group + * import. The suggested-prompts data also moves into `lib/ai/` for + * the same reason. + * + * Consumers (post-Phase-D): + * - `components/ai/AskPanel` — the workspace drawer / sidebar / + * fullscreen chat panel. + * - Nothing else. Both legacy `/ask` routes retire to redirects + * as part of Phase D. + * + * # Compact vs. full chrome + * + * The `compact` prop (default `false`) controls whether the shell + * renders its own `
` ("Ask the Commons" title + lede + share/ + * stop button row) and the page-height container, or just the inner + * chat-thread + input column. The AskPanel needs `compact=true` because + * it provides its own header chrome and a flex container that owns the + * height calculation. + * + * # Context prop + * + * Optional `context` carries workspace selection state (datasetId, + * datasetName, selection.subject / session / probe / stimulus / unit). + * + * Phase F (W7 fix from the 2026-05-16 audit): the context now IS + * forwarded to `/api/ask` via `DefaultChatTransport.body`. The route + * reads `body.context` and prepends a workspace-context system + * message so the model knows "the user is currently in dataset X + * looking at subject Y." Pre-fix, the prop was plumbed but + * underscored as unused — the AskPanel header line "Asking about: + * <dataset name>" was visual theater with zero API impact. + * + * # State management (unchanged from the pre-move version) + * + * The outer `AskShell` resolves the URL-hash conversation id via + * `useConversation`, then renders the inner `AskChat` keyed by + * `conversationId` so `useChat` reinitializes cleanly on "New chat". + * v5 of `@ai-sdk/react` — transport via `DefaultChatTransport`, send + * via `sendMessage({ text })`. See `lib/ai/use-conversation.ts` for + * the conversation-id + localStorage persistence layer. + */ +import { useChat } from '@ai-sdk/react'; +import { DefaultChatTransport, type UIMessage } from 'ai'; +import { useEffect, useMemo, useRef, useState } from 'react'; + +import { ChatInput } from '@/components/ai/ChatInput'; +import { ChatThread, type ThreadEntry } from '@/components/ai/ChatThread'; +import { ShareConversationButton } from '@/components/ai/ShareConversationButton'; +import { SuggestedPromptChips } from '@/components/ai/SuggestedPromptChips'; +import { SUGGESTED_PROMPTS } from '@/lib/ai/suggested-prompts'; +import { useConversation } from '@/lib/ai/use-conversation'; + +export interface AskShellContext { + datasetId?: string; + datasetName?: string; + /** + * The full 5-key selection from the workspace canvas, optional. + * Forwarded to `/api/ask` so the model knows which subject / + * session / probe / stimulus / unit the user is currently looking + * at when they ask a question. Absent → the chat falls back to + * dataset-only context. + */ + selectedSubjectId?: string; + selectedSessionId?: string; + selectedProbeId?: string; + selectedStimulusId?: string; + selectedUnitId?: string; +} + +export interface AskShellProps { + /** + * Workspace context. Forwarded to /api/ask via + * `DefaultChatTransport.body` so the server can prepend a + * workspace-context system message ("the user is in dataset X + * looking at subject Y"). Phase F (W7 fix) flips this from + * theater to wiring. + */ + context?: AskShellContext; + /** + * When true, render the inner chat column only (no shell header, + * no fixed-height container). Used by `AskPanel` which provides + * its own header + height management. + */ + compact?: boolean; + /** + * Optional prefill from elsewhere in the workspace (e.g. the + * data-grid bulk-actions bar). When this changes to a non-empty + * value, AskShell stages it into the input. If `autoSend` is + * true, the message fires immediately; otherwise it stays in the + * input for the user to review + send. + * + * Phase G integration with `lib/ai/ask-prefill-bus.ts`: AskPanel + * subscribes to the bus, opens the panel, and forwards the + * payload here via this prop. AskShell calls `onPrefillConsumed` + * after handling so the parent can clear its staged value and + * the same prefill doesn't fire twice on re-render. + */ + prefill?: { text: string; autoSend?: boolean } | null; + onPrefillConsumed?: () => void; +} + +/** + * Outer shell: resolves the conversation id (URL hash + localStorage + * restore) before handing off to the inner `AskChat`. We key + * `AskChat` by `conversationId` so: + * + * - On initial mount, the inner only renders once the id and + * `initialMessages` are settled (no hydration mismatch from + * touching window early). + * - On "New chat", `conversationId` changes → React unmounts and + * remounts the inner → `useChat` reinitializes from scratch + * with `messages: []`. + */ +export function AskShell({ + context, + compact = false, + prefill, + onPrefillConsumed, +}: AskShellProps = {}) { + const { + conversationId, + initialMessages, + persist, + startNewConversation, + shareUrl, + } = useConversation(); + + // Until the conversation hook has resolved, render a minimal + // placeholder. `conversationId` is the empty string before the + // mount effect fires. + if (!conversationId) { + return ( +
+ {!compact && ( +
+

+ Ask the Commons +

+
+ )} +
+ ); + } + + return ( + + ); +} + +type AskChatProps = { + conversationId: string; + initialMessages: UIMessage[]; + persist: (messages: UIMessage[]) => void; + onNewConversation: () => void; + shareUrl: string | null; + compact: boolean; + context: AskShellContext | undefined; + prefill: { text: string; autoSend?: boolean } | null; + onPrefillConsumed: (() => void) | undefined; +}; + +function AskChat({ + conversationId, + initialMessages, + persist, + onNewConversation, + shareUrl, + compact, + context, + prefill, + onPrefillConsumed, +}: AskChatProps) { + const [input, setInput] = useState(''); + const [errorBanner, setErrorBanner] = useState(null); + const [retryAt, setRetryAt] = useState(null); + + // Stringify context once per change so the transport rebuilds only + // when the user actually picks a different subject/session/etc. + // (URL state writes can fire several times per click; we don't want + // to thrash the transport.) + const contextKey = useMemo(() => JSON.stringify(context ?? null), [context]); + + // Transport built per-context — DefaultChatTransport's `body` + // option is merged into every POST to /api/ask. The server reads + // `body.context` and prepends a workspace-context system message + // so the model knows what selection the user is asking from. + // Phase F (W7 audit fix): pre-fix, context was theatre only. + const transport = useMemo( + () => + new DefaultChatTransport({ + api: '/api/ask', + body: context ? { context } : undefined, + }), + // eslint-disable-next-line react-hooks/exhaustive-deps + [contextKey], + ); + + const { messages, sendMessage, status, stop } = useChat({ + transport, + id: conversationId, + messages: initialMessages, + onError: (err) => { + const msg = err?.message ?? ''; + if (msg.includes('rate_limited') || msg.includes('429')) { + setErrorBanner( + "You've sent a lot of messages — wait a minute and try again.", + ); + setRetryAt(Date.now() + 60_000); + } else if (msg.includes('chat_disabled') || msg.includes('503')) { + setErrorBanner('Chat preview is not enabled in this environment.'); + } else { + setErrorBanner('Connection hiccup — try again.'); + } + }, + }); + + // Watchdog timer — see pre-move comment for the rationale (P0-B fix + // 2026-05-14). Carried over verbatim. + const STREAM_TIMEOUT_MS = 65_000; + const timeoutRef = useRef | null>(null); + const isStreamingNow = status === 'streaming' || status === 'submitted'; + useEffect(() => { + if (isStreamingNow) { + if (timeoutRef.current) clearTimeout(timeoutRef.current); + timeoutRef.current = setTimeout(() => { + stop(); + setErrorBanner( + 'The model took too long to answer. Try again with a more specific question, or wait a moment.', + ); + timeoutRef.current = null; + }, STREAM_TIMEOUT_MS); + return () => { + if (timeoutRef.current) { + clearTimeout(timeoutRef.current); + timeoutRef.current = null; + } + }; + } + if (timeoutRef.current) { + clearTimeout(timeoutRef.current); + timeoutRef.current = null; + } + return undefined; + }, [isStreamingNow, stop]); + + // Retry-after countdown. + useEffect(() => { + if (!retryAt) return; + const t = setInterval(() => { + if (Date.now() >= retryAt) { + setRetryAt(null); + setErrorBanner(null); + } + }, 1000); + return () => clearInterval(t); + }, [retryAt]); + + // Persist on every message change. The hook's debounce inside + // `useConversation` coalesces streaming tokens. + useEffect(() => { + persist(messages); + }, [messages, persist]); + + // Phase G — consume prefill events forwarded by AskPanel. Each + // distinct prefill payload (changed identity) fires once: stage + // text into the input, optionally auto-send, then notify the + // parent to clear its staged value. + // + // Guarded with `processedPrefillRef` so React 19's strict-mode + // double-effect doesn't double-send the same prefill. We capture + // a key based on the prefill payload itself; ref keeps "we already + // handled this" across re-renders without breaking the deps array. + const processedPrefillRef = useRef(null); + useEffect(() => { + if (!prefill) return; + if (processedPrefillRef.current === prefill) return; + processedPrefillRef.current = prefill; + if (prefill.autoSend) { + // Auto-send mode: fire the message directly. Don't stage in + // the input first — that would create a momentary "user is + // typing" flash before the send. The cleared input is the + // natural post-send state. + void sendMessage({ text: prefill.text }); + } else { + // Stage-only mode: drop the text into the input so the user + // can review + edit before sending. setState-in-effect is the + // right shape here — we're syncing a transient prop (prefill + // payload from the bus) into local input state. The + // processedPrefillRef guards against cascading re-renders. + // eslint-disable-next-line react-hooks/set-state-in-effect + setInput(prefill.text); + } + onPrefillConsumed?.(); + }, [prefill, sendMessage, onPrefillConsumed]); + + const entries: ThreadEntry[] = useMemo(() => { + const out: ThreadEntry[] = []; + for (const m of messages) { + const parts = m.parts as + | Array<{ + type: string; + text?: string; + toolName?: string; + input?: unknown; + output?: unknown; + }> + | undefined; + + if (!Array.isArray(parts)) continue; + + let buf = ''; + const toolCallsForMsg: Array<{ + toolName: string; + args: unknown; + result?: unknown; + }> = []; + + for (const p of parts) { + if (p.type === 'text' && typeof p.text === 'string') { + buf += p.text; + } else if (p.type.startsWith('tool-')) { + if (buf) { + out.push({ + kind: 'message', + role: m.role as 'user' | 'assistant', + content: buf, + }); + buf = ''; + } + const toolName = p.toolName ?? p.type.replace(/^tool-/, ''); + out.push({ kind: 'tool-call', toolName }); + if (m.role === 'assistant') { + toolCallsForMsg.push({ + toolName, + args: p.input, + result: p.output, + }); + } + } + } + if (buf) { + out.push({ + kind: 'message', + role: m.role as 'user' | 'assistant', + content: buf, + ...(m.role === 'assistant' && toolCallsForMsg.length > 0 + ? { toolCalls: toolCallsForMsg } + : {}), + }); + } else if (m.role === 'assistant' && toolCallsForMsg.length > 0) { + for (let i = out.length - 1; i >= 0; i--) { + const entry = out[i]!; + if (entry.kind === 'message' && entry.role === 'assistant') { + entry.toolCalls = [ + ...(entry.toolCalls ?? []), + ...toolCallsForMsg, + ]; + break; + } + } + } + } + return out; + }, [messages]); + + const lastUserQuestion = useMemo(() => { + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i]!; + if (m.role !== 'user') continue; + const parts = (m.parts ?? []) as Array<{ type: string; text?: string }>; + const text = parts + .filter((p) => p.type === 'text' && typeof p.text === 'string') + .map((p) => p.text) + .join(''); + if (text) return text; + } + return undefined; + }, [messages]); + + const chatUrl = + typeof window !== 'undefined' ? window.location.href : undefined; + + const isStreaming = status === 'streaming' || status === 'submitted'; + const isEmpty = messages.length === 0; + + const handleSubmit = () => { + const text = input.trim(); + if (!text || isStreaming) return; + setErrorBanner(null); + setInput(''); + void sendMessage({ text }); + }; + + const handleChipSelect = (prompt: string) => { + if (isStreaming) return; + setErrorBanner(null); + void sendMessage({ text: prompt }); + }; + + const handleStop = () => { + stop(); + if (timeoutRef.current) { + clearTimeout(timeoutRef.current); + timeoutRef.current = null; + } + setErrorBanner('Stopped. Try a different question or rephrase.'); + }; + + const hasAnyMessages = messages.length > 0; + + return ( +
+ {!compact && ( +
+
+
+

+ Ask the Commons +

+

+ Experimental preview. Ask about published NDI datasets in plain + English — counts, contents, contributors, anything in the + public catalog. +

+
+
+ + {isStreaming ? ( + + ) : ( + hasAnyMessages && ( + + ) + )} +
+
+
+ )} + + {isEmpty ? ( + + ) : ( + + )} + + {errorBanner && ( +
+ {errorBanner} +
+ )} + + + + {/* Compact mode: surface the "New chat" affordance inline since + the header is suppressed. Placed at the bottom of the column + so it doesn't compete with the input field for focus. */} + {compact && hasAnyMessages && !isStreaming && ( +
+ +
+ )} +
+ ); +} diff --git a/apps/web/components/ai/ChatInput.tsx b/apps/web/components/ai/ChatInput.tsx new file mode 100644 index 00000000..541d33ef --- /dev/null +++ b/apps/web/components/ai/ChatInput.tsx @@ -0,0 +1,67 @@ +'use client'; + +import { useRef, type FormEvent, type KeyboardEvent } from 'react'; + +type Props = { + value: string; + onChange: (v: string) => void; + onSubmit: () => void; + disabled?: boolean; + placeholder?: string; +}; + +/** + * Multi-line text input + Send button. + * + * - Enter sends (Shift+Enter newline). + * - Disabled state during in-flight stream + when rate-limited. + * - Auto-grows up to ~5 lines, then scrolls (avoids the bubble + * taking over the whole viewport on long pastes). + */ +export function ChatInput({ + value, + onChange, + onSubmit, + disabled = false, + placeholder = 'Ask about the NDI Commons catalog…', +}: Props) { + const ref = useRef(null); + + const handleKey = (e: KeyboardEvent) => { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + if (!disabled && value.trim().length > 0) onSubmit(); + } + }; + + const handleSubmit = (e: FormEvent) => { + e.preventDefault(); + if (!disabled && value.trim().length > 0) onSubmit(); + }; + + return ( +
+