From 20d341acb3ab6423b7aadbcc96e4be2ca0736ece Mon Sep 17 00:00:00 2001 From: Seth Davis Date: Sat, 30 May 2026 11:56:58 -0500 Subject: [PATCH] test(e2e): build out robust Playwright suite across the app Expand the E2E suite from auth/chat smoke tests to broad coverage: navigation/layout, dashboard, healthcheck, the /api/chat API boundary (401/400/403/404/405), agent tool rendering (note + card via mocked dynamic-tool SSE), chat error/retry/dismiss UX, and cross-user thread access control. Make auth state explicit and isolated: replace the shared-storageState setup project with a per-test fresh-user `authedPage` fixture so parallel runs never share thread state, and add `createAuthedContext` / `createThreadViaApi` helpers plus `tests/chat-mock.ts`. Run the suite on a dedicated port (E2E_PORT, default 7778) to avoid colliding with other local dev servers, and point both BETTER_AUTH base URLs at it so Better Auth trusts the origin. Add a test-only DISABLE_AUTH_RATE_LIMIT env toggle (off everywhere else) so rapid per-test sign-ups don't trip Better Auth's limiter. Fix stale /profile -> /dashboard assertions. 162 tests green across chromium, firefox, and webkit. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 4 +- app/lib/auth.server.ts | 6 +- app/lib/env.server.ts | 9 ++ playwright.config.ts | 45 ++++---- tests/api-chat.spec.ts | 85 +++++++++++++++ tests/auth-setup.ts | 18 ---- tests/auth.spec.ts | 18 ++-- tests/chat-mock.ts | 107 +++++++++++++++++++ tests/chat-tools.spec.ts | 142 +++++++++++++++++++++++++ tests/chat-ux.spec.ts | 120 +++++++++++++++++++++ tests/chat.spec.ts | 213 +++++++++++++++----------------------- tests/dashboard.spec.ts | 17 +++ tests/fixtures.ts | 125 +++++++++++++++++++--- tests/global-setup.ts | 10 +- tests/healthcheck.spec.ts | 12 +++ tests/home.spec.ts | 26 ++++- tests/navigation.spec.ts | 117 +++++++++++++++++++++ tests/security.spec.ts | 53 ++++++++++ 18 files changed, 930 insertions(+), 197 deletions(-) create mode 100644 tests/api-chat.spec.ts delete mode 100644 tests/auth-setup.ts create mode 100644 tests/chat-mock.ts create mode 100644 tests/chat-tools.spec.ts create mode 100644 tests/chat-ux.spec.ts create mode 100644 tests/dashboard.spec.ts create mode 100644 tests/healthcheck.spec.ts create mode 100644 tests/navigation.spec.ts create mode 100644 tests/security.spec.ts diff --git a/CLAUDE.md b/CLAUDE.md index 2ca8e8c..cf79191 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -117,7 +117,9 @@ In-memory sliding window in `app/lib/rate-limit.server.ts`. Used for chat (20/mi **Unit tests** use Vitest (`bun run test`). Test files live alongside source files as `*.test.ts`. Modules that import server-side dependencies (auth, Prisma) need `vi.mock()` to avoid env validation side effects. -**E2E tests** use Playwright (`bun run test:e2e`) in `tests/`. Auth setup project runs first and saves `storageState` to `test-results/.auth/user.json` -- all browser projects reuse this so tests don't re-login. Test fixtures in `tests/fixtures.ts` export `test` (with `authedPage` fixture), `expect`, and `TEST_USER`. Chat tests mock `/api/chat` with canned SSE responses (no AI service needed). +**E2E tests** use Playwright (`bun run test:e2e`) in `tests/`, covering auth, navigation, dashboard, healthcheck, the chat flow, agent tool rendering, chat error UX, the `/api/chat` API boundary, and cross-user thread access control. They run against a dedicated dev server on port `7778` (override with `E2E_PORT`) so they never collide with `bun run dev` on 5173; the `webServer` config also points `BETTER_AUTH_BASE_URL`/`VITE_BETTER_AUTH_BASE_URL` at that port and sets `DISABLE_AUTH_RATE_LIMIT=true` plus a dummy `ANTHROPIC_API_KEY`. + +Auth is explicit per test: the `authedPage` fixture in `tests/fixtures.ts` signs up a brand-new isolated user on demand (so every test starts with zero threads and parallel runs never share state), while a plain `page` stays logged out. `globalSetup` only ensures the seed users (Alice, Bob) exist for tests that log in as them. Fixtures also export `createAuthedContext` and `createThreadViaApi` for multi-user scenarios. Chat tests mock `/api/chat` with canned SSE responses (no AI service needed); tool-rendering tests stream `dynamic-tool` parts via helpers in `tests/chat-mock.ts`. ## Conventions diff --git a/app/lib/auth.server.ts b/app/lib/auth.server.ts index cde1503..2032ee7 100644 --- a/app/lib/auth.server.ts +++ b/app/lib/auth.server.ts @@ -40,9 +40,11 @@ export const auth = betterAuth({ }, plugins: [admin({ defaultRole: 'USER' })], // Better Auth's built-in rate limiter. Defaults are off in non-prod; - // explicitly enable so dev and CI exercise the same limits as prod. + // explicitly enable so dev and CI exercise the same limits as prod. The + // E2E test server opts out via DISABLE_AUTH_RATE_LIMIT so it can create + // many sessions quickly. rateLimit: { - enabled: true, + enabled: !env.DISABLE_AUTH_RATE_LIMIT, // 10s sliding window, 100 req/window per IP across all auth endpoints. window: 10, max: 100, diff --git a/app/lib/env.server.ts b/app/lib/env.server.ts index 57e304d..eb1d04f 100644 --- a/app/lib/env.server.ts +++ b/app/lib/env.server.ts @@ -29,6 +29,15 @@ const envSchema = z.object({ NODE_ENV: z .enum(['development', 'production', 'test']) .default('development'), + /** + * Disables Better Auth's rate limiter. Intended only for the E2E test + * server, which creates many sessions quickly; unset everywhere else so + * dev, CI, and prod keep the production limits. + */ + DISABLE_AUTH_RATE_LIMIT: z + .enum(['true', 'false']) + .optional() + .transform((value) => value === 'true'), }); function validateEnv() { diff --git a/playwright.config.ts b/playwright.config.ts index 13b7827..d94acea 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,5 +1,9 @@ import { defineConfig, devices } from '@playwright/test'; -import { STORAGE_STATE } from './tests/fixtures'; + +// Iridium's dev port 5173 is often occupied by other local projects, so the +// E2E suite runs the app on a dedicated port. Override with E2E_PORT if needed. +const PORT = Number(process.env.E2E_PORT ?? 7778); +const BASE_URL = `http://localhost:${PORT}`; export default defineConfig({ globalSetup: './tests/global-setup.ts', @@ -10,47 +14,52 @@ export default defineConfig({ workers: process.env.CI ? 1 : undefined, reporter: 'html', use: { - baseURL: 'http://localhost:5173', + baseURL: BASE_URL, navigationTimeout: 60000, trace: 'on-first-retry', }, + // Auth is explicit per test: the `authedPage` fixture creates a fresh, + // isolated user on demand, while a plain `page` stays logged out. No global + // storageState or setup project is needed; `globalSetup` only ensures the + // named seed users (Alice, Bob) exist for the tests that log in as them. projects: [ - // Auth setup — runs first, saves storageState for other projects - { - name: 'setup', - testMatch: /auth-setup\.ts/, - }, { name: 'chromium', use: { ...devices['Desktop Chrome'], - storageState: STORAGE_STATE, }, - dependencies: ['setup'], - testIgnore: /auth-setup\.ts/, }, { name: 'firefox', use: { ...devices['Desktop Firefox'], - storageState: STORAGE_STATE, }, - dependencies: ['setup'], - testIgnore: /auth-setup\.ts/, }, { name: 'webkit', use: { ...devices['Desktop Safari'], - storageState: STORAGE_STATE, }, - dependencies: ['setup'], - testIgnore: /auth-setup\.ts/, }, ], webServer: { - command: 'bun run dev', - url: 'http://localhost:5173', + command: `bun run dev --port ${PORT}`, + url: BASE_URL, reuseExistingServer: !process.env.CI, + // The suite mocks /api/chat, so the real Anthropic key is never used — + // env validation just needs a non-empty value for the server to boot. + // The two BETTER_AUTH base URLs must match the test port: the server + // one drives Better Auth's trusted origins, and the VITE_ one is baked + // into the browser auth client — if it points elsewhere, sign-in POSTs + // to the wrong origin and silently hangs. + env: { + ANTHROPIC_API_KEY: + process.env.ANTHROPIC_API_KEY || 'sk-ant-e2e-dummy-key', + BETTER_AUTH_BASE_URL: BASE_URL, + VITE_BETTER_AUTH_BASE_URL: BASE_URL, + // The suite creates a fresh user per test; the prod-tuned auth rate + // limiter would otherwise reject the rapid sign-ups with 429s. + DISABLE_AUTH_RATE_LIMIT: 'true', + }, }, }); diff --git a/tests/api-chat.spec.ts b/tests/api-chat.spec.ts new file mode 100644 index 0000000..7b407f6 --- /dev/null +++ b/tests/api-chat.spec.ts @@ -0,0 +1,85 @@ +import { + test, + expect, + createAuthedContext, + createThreadViaApi, +} from './fixtures'; + +/** + * API-level checks for the chat endpoint. These exercise the auth/ownership + * boundary that runs before any tokens are spent, so no AI service is needed. + */ + +function validBody(id = 'some-thread') { + return { + id, + messages: [ + { id: 'm1', role: 'user', parts: [{ type: 'text', text: 'hi' }] }, + ], + }; +} + +test.describe('POST /api/chat', () => { + test('rejects write methods other than POST with 405', async ({ + request, + }) => { + expect((await request.delete('/api/chat')).status()).toBe(405); + expect( + (await request.fetch('/api/chat', { method: 'PUT' })).status(), + ).toBe(405); + }); + + test('rejects GET (no loader on the endpoint)', async ({ request }) => { + const res = await request.get('/api/chat'); + expect(res.ok()).toBeFalsy(); + expect(res.status()).toBe(400); + }); + + test('returns 401 when unauthenticated', async ({ request }) => { + const res = await request.post('/api/chat', { data: validBody() }); + expect(res.status()).toBe(401); + }); + + test('returns 400 for an invalid request body', async ({ request }) => { + const res = await request.post('/api/chat', { data: { nope: true } }); + expect(res.status()).toBe(400); + }); + + test('returns 404 for a thread that does not exist', async ({ + browser, + baseURL, + }) => { + const context = await createAuthedContext(browser, baseURL!, 'chat404'); + try { + const res = await context.request.post('/api/chat', { + data: validBody('thread-does-not-exist'), + }); + expect(res.status()).toBe(404); + } finally { + await context.close(); + } + }); + + test("returns 403 for another user's thread", async ({ + browser, + baseURL, + }) => { + const owner = await createAuthedContext(browser, baseURL!, 'owner'); + const attacker = await createAuthedContext( + browser, + baseURL!, + 'attacker', + ); + try { + const threadId = await createThreadViaApi(owner); + + const res = await attacker.request.post('/api/chat', { + data: validBody(threadId), + }); + expect(res.status()).toBe(403); + } finally { + await owner.close(); + await attacker.close(); + } + }); +}); diff --git a/tests/auth-setup.ts b/tests/auth-setup.ts deleted file mode 100644 index a3a5722..0000000 --- a/tests/auth-setup.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { test as setup } from '@playwright/test'; -import { STORAGE_STATE, TEST_USER } from './fixtures'; - -/** - * Runs once before all test projects. Signs in as the test user - * and saves the authenticated browser state to disk so every - * subsequent test reuses the session cookie without re-logging in. - */ -setup('authenticate', async ({ page }) => { - await page.goto('/login'); - await page.getByPlaceholder('name@example.com').fill(TEST_USER.email); - await page.getByPlaceholder('Your password').fill(TEST_USER.password); - await page.getByRole('button', { name: 'Login' }).click(); - - await page.waitForURL(/\/profile/); - - await page.context().storageState({ path: STORAGE_STATE }); -}); diff --git a/tests/auth.spec.ts b/tests/auth.spec.ts index e51c7b1..6b552f0 100644 --- a/tests/auth.spec.ts +++ b/tests/auth.spec.ts @@ -1,7 +1,7 @@ import { test, expect, TEST_USER } from './fixtures'; test.describe('Login', () => { - test('logs in with valid credentials and redirects to profile', async ({ + test('logs in with valid credentials and redirects to dashboard', async ({ page, }) => { await page.goto('/login'); @@ -9,8 +9,8 @@ test.describe('Login', () => { await page.getByPlaceholder('Your password').fill(TEST_USER.password); await page.getByRole('button', { name: 'Login' }).click(); - await expect(page).toHaveURL(/\/profile/); - await expect(page.getByText(TEST_USER.email)).toBeVisible(); + await expect(page).toHaveURL(/\/dashboard/); + await expect(page.getByText('Hello Dashboard!')).toBeVisible(); }); test('shows error for invalid credentials', async ({ page }) => { @@ -66,7 +66,9 @@ test.describe('Registration', () => { await expect(page.getByText('Name is required')).toBeVisible(); }); - test('registers a new user and redirects to profile', async ({ page }) => { + test('registers a new user and redirects to dashboard', async ({ + page, + }) => { const unique = `e2e-${Date.now()}@test.com`; await page.goto('/login'); await page.getByRole('radio', { name: 'Register' }).check(); @@ -74,10 +76,8 @@ test.describe('Registration', () => { await page.getByPlaceholder('name@example.com').fill(unique); await page.getByPlaceholder('Your password').fill('password123'); await page.getByRole('button', { name: 'Register' }).click(); - await expect( - page.getByRole('heading', { name: 'Profile' }), - ).toBeVisible(); - await expect(page.getByText(unique)).toBeVisible(); + await expect(page).toHaveURL(/\/dashboard/); + await expect(page.getByText('Hello Dashboard!')).toBeVisible(); }); test('shows error when registering with existing email', async ({ @@ -108,7 +108,7 @@ test.describe('Logout', () => { test('protected pages redirect to login when logged out', async ({ page, }) => { - await page.goto('/profile'); + await page.goto('/dashboard'); await expect(page).toHaveURL(/\/login/); }); }); diff --git a/tests/chat-mock.ts b/tests/chat-mock.ts new file mode 100644 index 0000000..9d57b5d --- /dev/null +++ b/tests/chat-mock.ts @@ -0,0 +1,107 @@ +import type { Page, Route as PwRoute } from '@playwright/test'; + +/** + * Helpers for faking the `/api/chat` Server-Sent Events stream so E2E tests + * never call a real AI provider. The chunk shapes mirror the Vercel AI SDK + * v6 UI message stream (see `node_modules/ai/dist/index.d.ts`): text parts use + * `text-*` chunks; tool parts use `tool-input-*` / `tool-output-available`, + * which `useChat` assembles into a `tool-` message part. + */ + +export const SSE_HEADERS = { + 'content-type': 'text/event-stream', + 'cache-control': 'no-cache', + connection: 'keep-alive', + 'x-vercel-ai-ui-message-stream': 'v1', +}; + +/** Serialize an array of UI message stream chunks into an SSE body. */ +export function sseBody(chunks: Record[]): string { + return ( + chunks.map((c) => `data: ${JSON.stringify(c)}\n\n`).join('') + + 'data: [DONE]\n\n' + ); +} + +/** Chunks that stream a single plain-text assistant reply. */ +export function textReplyChunks(text: string, messageId = 'mock-msg-1') { + return [ + { type: 'start', messageId }, + { type: 'start-step' }, + { type: 'text-start', id: 'txt-1' }, + { type: 'text-delta', id: 'txt-1', delta: text }, + { type: 'text-end', id: 'txt-1' }, + { type: 'finish-step' }, + { type: 'finish', finishReason: 'stop' }, + ]; +} + +/** + * Chunks that stream a tool call which resolves to `output`. The `dynamic: true` + * flag makes the SDK assemble a `dynamic-tool` part that carries `toolName` — + * which is what the thread view reads to pick the right tool renderer (the + * VoltAgent backend emits dynamic tool parts too). + */ +export function toolCallChunks({ + toolName, + input = {}, + output, + messageId = 'mock-tool-msg', + toolCallId = 'tool-call-1', +}: { + toolName: string; + input?: unknown; + output: unknown; + messageId?: string; + toolCallId?: string; +}) { + return [ + { type: 'start', messageId }, + { type: 'start-step' }, + { type: 'tool-input-start', toolCallId, toolName, dynamic: true }, + { + type: 'tool-input-available', + toolCallId, + toolName, + input, + dynamic: true, + }, + { type: 'tool-output-available', toolCallId, output, dynamic: true }, + { type: 'finish-step' }, + { type: 'finish', finishReason: 'stop' }, + ]; +} + +/** Intercept POST /api/chat and reply with a canned SSE stream of `chunks`. */ +export async function mockChatStream( + page: Page, + chunks: Record[], +) { + await page.route('**/api/chat', async (route: PwRoute) => { + await route.fulfill({ + status: 200, + headers: SSE_HEADERS, + body: sseBody(chunks), + }); + }); +} + +/** Intercept POST /api/chat and reply with a plain-text assistant message. */ +export async function mockChatReply( + page: Page, + text: string, + messageId?: string, +) { + await mockChatStream(page, textReplyChunks(text, messageId)); +} + +/** Intercept POST /api/chat and fail it, to exercise the error UI. */ +export async function mockChatError(page: Page, status = 500) { + await page.route('**/api/chat', async (route: PwRoute) => { + await route.fulfill({ + status, + contentType: 'application/json', + body: JSON.stringify({ error: 'Something went wrong.' }), + }); + }); +} diff --git a/tests/chat-tools.spec.ts b/tests/chat-tools.spec.ts new file mode 100644 index 0000000..dad87e3 --- /dev/null +++ b/tests/chat-tools.spec.ts @@ -0,0 +1,142 @@ +import { test, expect } from './fixtures'; +import { mockChatStream, toolCallChunks } from './chat-mock'; + +/** + * Verifies the tool-driven generative UI: the agent returns structured tool + * output and the thread view renders a rich component (note summary or card) + * instead of plain text. The /api/chat stream is mocked with dynamic-tool + * parts that mirror what the VoltAgent backend emits. + */ + +async function openNewThread(page: import('@playwright/test').Page) { + await page.goto('/chat'); + await page.getByRole('button', { name: 'New Thread' }).click(); + await expect(page).toHaveURL(/\/chat\/.+/); +} + +async function send(page: import('@playwright/test').Page, text: string) { + await page.getByPlaceholder('Your message here...').fill(text); + await page.getByRole('button', { name: 'Send' }).click(); +} + +test.describe('Agent tool rendering', () => { + test('create_note renders a saved-note confirmation', async ({ + authedPage: page, + }) => { + await openNewThread(page); + await mockChatStream( + page, + toolCallChunks({ + toolName: 'create_note', + input: { title: 'Groceries', content: 'milk, eggs' }, + output: { id: 'n1', title: 'Groceries', content: 'milk, eggs' }, + }), + ); + + await send(page, 'Save a note'); + + await expect(page.getByText('Creating note')).toBeVisible(); + await expect(page.getByText('Saved: Groceries')).toBeVisible(); + }); + + test('list_notes renders the returned notes', async ({ + authedPage: page, + }) => { + await openNewThread(page); + await mockChatStream( + page, + toolCallChunks({ + toolName: 'list_notes', + output: { + notes: [ + { id: '1', title: 'Alpha', content: 'first note body' }, + { id: '2', title: 'Beta', content: 'second note body' }, + ], + }, + }), + ); + + await send(page, 'List my notes'); + + await expect(page.getByText('Listing notes')).toBeVisible(); + await expect(page.getByText('Alpha')).toBeVisible(); + await expect(page.getByText('Beta')).toBeVisible(); + }); + + test('render_card (info) renders an info card', async ({ + authedPage: page, + }) => { + await openNewThread(page); + await mockChatStream( + page, + toolCallChunks({ + toolName: 'render_card', + output: { + variant: 'info', + title: 'Did you know?', + description: 'Iridium is dense.', + items: ['point one', 'point two'], + }, + }), + ); + + await send(page, 'Show me a fact'); + + await expect( + page.getByRole('heading', { name: 'Did you know?' }), + ).toBeVisible(); + await expect(page.getByText('Iridium is dense.')).toBeVisible(); + await expect(page.getByText('point one')).toBeVisible(); + }); + + test('render_card (steps) renders an ordered steps card', async ({ + authedPage: page, + }) => { + await openNewThread(page); + await mockChatStream( + page, + toolCallChunks({ + toolName: 'render_card', + output: { + variant: 'steps', + title: 'How to deploy', + steps: ['Build the app', 'Push to Railway'], + }, + }), + ); + + await send(page, 'How do I deploy?'); + + await expect( + page.getByRole('heading', { name: 'How to deploy' }), + ).toBeVisible(); + await expect(page.getByText('Build the app')).toBeVisible(); + await expect(page.getByText('Push to Railway')).toBeVisible(); + }); + + test('render_card (pros_cons) renders both columns', async ({ + authedPage: page, + }) => { + await openNewThread(page); + await mockChatStream( + page, + toolCallChunks({ + toolName: 'render_card', + output: { + variant: 'pros_cons', + title: 'Bun vs Node', + pros: ['Fast installs'], + cons: ['Younger ecosystem'], + }, + }), + ); + + await send(page, 'Compare Bun and Node'); + + await expect( + page.getByRole('heading', { name: 'Bun vs Node' }), + ).toBeVisible(); + await expect(page.getByText('Fast installs')).toBeVisible(); + await expect(page.getByText('Younger ecosystem')).toBeVisible(); + }); +}); diff --git a/tests/chat-ux.spec.ts b/tests/chat-ux.spec.ts new file mode 100644 index 0000000..f5834fa --- /dev/null +++ b/tests/chat-ux.spec.ts @@ -0,0 +1,120 @@ +import type { Page } from '@playwright/test'; +import { test, expect } from './fixtures'; +import { mockChatError, mockChatReply } from './chat-mock'; + +async function openNewThread(page: Page) { + await page.goto('/chat'); + await page.getByRole('button', { name: 'New Thread' }).click(); + await expect(page).toHaveURL(/\/chat\/.+/); +} + +async function send(page: Page, text: string) { + await page.getByPlaceholder('Your message here...').fill(text); + await page.getByRole('button', { name: 'Send' }).click(); +} + +test.describe('Chat input behaviour', () => { + test('Enter key sends the message', async ({ authedPage: page }) => { + await openNewThread(page); + await mockChatReply(page, 'Reply via Enter'); + + const input = page.getByPlaceholder('Your message here...'); + await input.fill('hello there'); + await input.press('Enter'); + + await expect(page.getByText('hello there')).toBeVisible(); + await expect(page.getByText('Reply via Enter')).toBeVisible(); + }); + + test('does not send an empty message', async ({ authedPage: page }) => { + await openNewThread(page); + await page.getByRole('button', { name: 'Send' }).click(); + await expect(page.getByText('No messages yet')).toBeVisible(); + }); + + test('Stop is disabled and Send enabled at rest', async ({ + authedPage: page, + }) => { + await openNewThread(page); + await expect(page.getByRole('button', { name: 'Stop' })).toBeDisabled(); + await expect(page.getByRole('button', { name: 'Send' })).toBeEnabled(); + }); +}); + +test.describe('Chat preset prompts', () => { + const PRESETS = [ + 'Summarize', + 'Explain', + 'Pros & Cons', + 'Next Steps', + 'My Notes', + 'Save Note', + ]; + + test('all preset buttons are shown and enabled', async ({ + authedPage: page, + }) => { + await openNewThread(page); + for (const label of PRESETS) { + await expect( + page.getByRole('button', { name: label }), + ).toBeEnabled(); + } + }); + + test('clicking a preset sends it and shows the reply', async ({ + authedPage: page, + }) => { + await openNewThread(page); + await mockChatReply(page, 'Here is your summary'); + await page.getByRole('button', { name: 'Summarize' }).click(); + await expect(page.getByText('Here is your summary')).toBeVisible(); + }); +}); + +test.describe('Chat error handling', () => { + test('shows an error alert when the request fails', async ({ + authedPage: page, + }) => { + await openNewThread(page); + await mockChatError(page, 500); + await send(page, 'this will fail'); + + const alert = page.getByRole('alert'); + await expect(alert).toBeVisible(); + await expect(alert).toContainText('Something went wrong.'); + await expect( + alert.getByRole('button', { name: 'Retry' }), + ).toBeVisible(); + await expect( + alert.getByRole('button', { name: 'Dismiss' }), + ).toBeVisible(); + }); + + test('Dismiss clears the error alert', async ({ authedPage: page }) => { + await openNewThread(page); + await mockChatError(page, 500); + await send(page, 'this will fail'); + + const alert = page.getByRole('alert'); + await expect(alert).toBeVisible(); + await alert.getByRole('button', { name: 'Dismiss' }).click(); + await expect(alert).toHaveCount(0); + }); + + test('Retry re-sends and can succeed', async ({ authedPage: page }) => { + await openNewThread(page); + await mockChatError(page, 500); + await send(page, 'flaky message'); + + await expect(page.getByRole('alert')).toBeVisible(); + + // Swap the mock to succeed, then retry. + await page.unroute('**/api/chat'); + await mockChatReply(page, 'Recovered reply'); + await page.getByRole('button', { name: 'Retry' }).click(); + + await expect(page.getByText('Recovered reply')).toBeVisible(); + await expect(page.getByRole('alert')).toHaveCount(0); + }); +}); diff --git a/tests/chat.spec.ts b/tests/chat.spec.ts index ea8ff96..8bb47de 100644 --- a/tests/chat.spec.ts +++ b/tests/chat.spec.ts @@ -1,57 +1,35 @@ -import type { Page, Route as PwRoute } from '@playwright/test'; +import type { Route as PwRoute } from '@playwright/test'; import { test, expect } from './fixtures'; +import { SSE_HEADERS, mockChatReply, sseBody } from './chat-mock'; -// --------------------------------------------------------------------------- -// Mock helpers — fake the /api/chat SSE stream so no AI service is needed -// --------------------------------------------------------------------------- - -const SSE_HEADERS = { - 'content-type': 'text/event-stream', - 'cache-control': 'no-cache', - connection: 'keep-alive', - 'x-vercel-ai-ui-message-stream': 'v1', -}; - -/** Build an SSE payload from an array of UI message stream chunks. */ -function sseBody(chunks: Record[]): string { - return ( - chunks.map((c) => `data: ${JSON.stringify(c)}\n\n`).join('') + - 'data: [DONE]\n\n' - ); -} +/** + * Each test runs as a fresh, isolated user (see the `authedPage` fixture), so + * the thread sidebar starts empty and counts/links are deterministic. Threads + * are counted by their `link` role — the "No threads found" empty state is a + * listitem with no link, so it never inflates the count. + */ -/** Standard chunks that stream a plain text reply. */ -function textReplyChunks(text: string, messageId = 'mock-msg-1') { - return [ - { type: 'start', messageId }, - { type: 'start-step' }, - { type: 'text-start', id: 'txt-1' }, - { type: 'text-delta', id: 'txt-1', delta: text }, - { type: 'text-end', id: 'txt-1' }, - { type: 'finish-step' }, - { type: 'finish', finishReason: 'stop' }, - ]; -} +const conversations = (page: import('@playwright/test').Page) => + page.getByRole('navigation', { name: 'Conversations' }); -/** Intercept POST /api/chat and reply with a canned SSE stream. */ -async function mockChatApi(page: Page, text: string, messageId?: string) { - await page.route('**/api/chat', async (route: PwRoute) => { - await route.fulfill({ - status: 200, - headers: SSE_HEADERS, - body: sseBody(textReplyChunks(text, messageId)), - }); - }); +/** Path portion (e.g. /chat/abc123) of the currently open thread URL. */ +function threadPath(url: string): string { + return new URL(url).pathname; } -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - test.describe('Chat', () => { - test('can create a new thread', async ({ authedPage: page }) => { + test('starts with an empty thread list and prompt to pick a thread', async ({ + authedPage: page, + }) => { await page.goto('/chat'); + await expect(page.getByText('Pick a thread!')).toBeVisible(); + await expect(page.getByText('No threads found')).toBeVisible(); + await expect(conversations(page).getByRole('link')).toHaveCount(0); + }); + + test('can create a new thread', async ({ authedPage: page }) => { + await page.goto('/chat'); await page.getByRole('button', { name: 'New Thread' }).click(); await expect(page).toHaveURL(/\/chat\/.+/); @@ -60,7 +38,17 @@ test.describe('Chat', () => { await expect(page.getByText('No messages yet')).toBeVisible(); }); - test('sending a message shows it in the chat', async ({ + test('new thread appears in the sidebar', async ({ authedPage: page }) => { + await page.goto('/chat'); + await expect(conversations(page).getByRole('link')).toHaveCount(0); + + await page.getByRole('button', { name: 'New Thread' }).click(); + await expect(page).toHaveURL(/\/chat\/.+/); + + await expect(conversations(page).getByRole('link')).toHaveCount(1); + }); + + test('sending a message shows the user message and AI reply', async ({ authedPage: page, }) => { await page.goto('/chat'); @@ -68,26 +56,23 @@ test.describe('Chat', () => { await expect(page).toHaveURL(/\/chat\/.+/); const reply = 'Hello from mock AI!'; - await mockChatApi(page, reply); + await mockChatReply(page, reply); - const input = page.getByPlaceholder('Your message here...'); - await input.fill('Hi there'); + await page.getByPlaceholder('Your message here...').fill('Hi there'); await page.getByRole('button', { name: 'Send' }).click(); - // User message appears await expect(page.getByText('Hi there')).toBeVisible(); - // AI reply streams in await expect(page.getByText(reply)).toBeVisible(); }); - test('AI response streams back and renders', async ({ + test('AI response streams back incrementally', async ({ authedPage: page, }) => { await page.goto('/chat'); await page.getByRole('button', { name: 'New Thread' }).click(); await expect(page).toHaveURL(/\/chat\/.+/); - // Stream two separate deltas to verify incremental rendering + // Stream two separate deltas to verify incremental concatenation. await page.route('**/api/chat', async (route: PwRoute) => { await route.fulfill({ status: 200, @@ -96,16 +81,8 @@ test.describe('Chat', () => { { type: 'start', messageId: 'msg-stream' }, { type: 'start-step' }, { type: 'text-start', id: 'txt-s' }, - { - type: 'text-delta', - id: 'txt-s', - delta: 'first chunk ', - }, - { - type: 'text-delta', - id: 'txt-s', - delta: 'second chunk', - }, + { type: 'text-delta', id: 'txt-s', delta: 'first chunk ' }, + { type: 'text-delta', id: 'txt-s', delta: 'second chunk' }, { type: 'text-end', id: 'txt-s' }, { type: 'finish-step' }, { type: 'finish', finishReason: 'stop' }, @@ -113,97 +90,77 @@ test.describe('Chat', () => { }); }); - const input = page.getByPlaceholder('Your message here...'); - await input.fill('Stream test'); + await page.getByPlaceholder('Your message here...').fill('Stream test'); await page.getByRole('button', { name: 'Send' }).click(); - // Both deltas should be concatenated await expect(page.getByText('first chunk second chunk')).toBeVisible(); }); - test('thread appears in the sidebar after creation', async ({ - authedPage: page, - }) => { - await page.goto('/chat'); - - const sidebar = page.getByRole('navigation', { - name: 'Conversations', - }); - - // Count threads before - const beforeCount = await sidebar.getByRole('listitem').count(); - - await page.getByRole('button', { name: 'New Thread' }).click(); - await expect(page).toHaveURL(/\/chat\/.+/); - - // Sidebar should now have one more thread - await expect(sidebar.getByRole('listitem')).toHaveCount( - beforeCount + 1, - ); - }); - test('thread can be deleted', async ({ authedPage: page }) => { await page.goto('/chat'); await page.getByRole('button', { name: 'New Thread' }).click(); await expect(page).toHaveURL(/\/chat\/.+/); + const path = threadPath(page.url()); - const sidebar = page.getByRole('navigation', { - name: 'Conversations', - }); - - // The new thread should be in the sidebar - const threadCount = await sidebar.getByRole('listitem').count(); - expect(threadCount).toBeGreaterThanOrEqual(1); + const sidebar = conversations(page); + await expect(sidebar.getByRole('link')).toHaveCount(1); - // Delete the first thread — button is hidden until hover - const firstThread = sidebar.getByRole('listitem').first(); - await firstThread.hover(); - await firstThread - .getByRole('button', { name: 'Delete thread' }) - .click(); + // The delete button is revealed on hover of the thread's row. + const row = sidebar + .getByRole('listitem') + .filter({ has: page.locator(`a[href="${path}"]`) }); + await row.hover(); + await row.getByRole('button', { name: 'Delete thread' }).click(); - // Confirm deletion in the dialog + // Confirm in the modal. const dialog = page.getByRole('dialog'); await expect(dialog).toBeVisible(); await dialog.getByRole('button', { name: 'Delete' }).click(); - // Should redirect to /chat and have one fewer thread + // Redirects back to the chat index, and the thread is gone. await expect(page).toHaveURL(/\/chat$/); - await expect(sidebar.getByRole('listitem')).toHaveCount( - threadCount - 1, - ); + await expect(sidebar.getByRole('link')).toHaveCount(0); + await expect(page.getByText('No threads found')).toBeVisible(); }); - test('navigating between threads loads correct messages', async ({ + test('can navigate between multiple threads', async ({ authedPage: page, }) => { - // Create two threads await page.goto('/chat'); + + // Create the first thread. await page.getByRole('button', { name: 'New Thread' }).click(); await expect(page).toHaveURL(/\/chat\/.+/); - const firstThreadUrl = page.url(); - - // Send a message in the first thread - await mockChatApi(page, 'Reply to thread one'); - const input = page.getByPlaceholder('Your message here...'); - await input.fill('Message in thread one'); - await page.getByRole('button', { name: 'Send' }).click(); - await expect(page.getByText('Reply to thread one')).toBeVisible(); + const firstPath = threadPath(page.url()); + await expect(page.getByText('No messages yet')).toBeVisible(); - // Create a second thread - await page.unrouteAll({ behavior: 'wait' }); + // Create a second thread — wait for the path to actually change since + // we are already on a /chat/ URL. await page.getByRole('button', { name: 'New Thread' }).click(); - await page.waitForURL(/\/chat\/.+/); - // Make sure we're on a different thread - expect(page.url()).not.toBe(firstThreadUrl); + await page.waitForURL( + (url) => + /\/chat\/.+/.test(url.pathname) && url.pathname !== firstPath, + ); + const secondPath = threadPath(page.url()); + + const sidebar = conversations(page); + await expect(sidebar.getByRole('link')).toHaveCount(2); + + // The open thread's link is marked current; the other is not. + await expect( + sidebar.locator(`a[href="${secondPath}"]`), + ).toHaveAttribute('aria-current', 'page'); + await expect( + sidebar.locator(`a[href="${firstPath}"]`), + ).not.toHaveAttribute('aria-current', 'page'); + + // Switch back to the first thread. + await sidebar.locator(`a[href="${firstPath}"]`).click(); + await expect(page).toHaveURL(new RegExp(`${firstPath}$`)); await expect(page.getByText('No messages yet')).toBeVisible(); - - // Navigate back to the first thread - const sidebar = page.getByRole('navigation', { - name: 'Conversations', - }); - // The first thread should still have its message when we reload it - await sidebar.getByRole('link').first().click(); - await expect(page.getByText('Message in thread one')).toBeVisible(); + await expect(sidebar.locator(`a[href="${firstPath}"]`)).toHaveAttribute( + 'aria-current', + 'page', + ); }); }); diff --git a/tests/dashboard.spec.ts b/tests/dashboard.spec.ts new file mode 100644 index 0000000..1cfe67e --- /dev/null +++ b/tests/dashboard.spec.ts @@ -0,0 +1,17 @@ +import { test, expect } from './fixtures'; + +test.describe('Dashboard', () => { + test('renders for an authenticated user', async ({ authedPage: page }) => { + await page.goto('/dashboard'); + await expect(page).toHaveURL(/\/dashboard$/); + await expect(page.getByText('Hello Dashboard!')).toBeVisible(); + }); + + test('redirects to login when logged out', async ({ page }) => { + await page.goto('/dashboard'); + await expect(page).toHaveURL(/\/login/); + await expect( + page.getByRole('heading', { name: 'Authenticate' }), + ).toBeVisible(); + }); +}); diff --git a/tests/fixtures.ts b/tests/fixtures.ts index e6237dd..55d8f07 100644 --- a/tests/fixtures.ts +++ b/tests/fixtures.ts @@ -1,10 +1,10 @@ -import { test as base, expect } from '@playwright/test'; -import path from 'node:path'; - -export const STORAGE_STATE = path.join( - import.meta.dirname, - '../test-results/.auth/user.json', -); +import { + test as base, + expect, + type Browser, + type BrowserContext, + type Page, +} from '@playwright/test'; export const TEST_USER = { name: 'Alice', @@ -12,17 +12,110 @@ export const TEST_USER = { password: 'password123', }; +/** Second seeded user, used for cross-user authorization tests. */ +export const TEST_USER_BOB = { + name: 'Bob', + email: 'bob@iridium.dev', + password: 'password123', +}; + +/** Where the auth form sends users on success. */ +export const REDIRECT_PATH = '/dashboard'; + +/** Sign in through the login form and wait for the post-login redirect. */ +export async function loginViaUI( + page: Page, + user: { email: string; password: string } = TEST_USER, +) { + await page.goto('/login'); + await page.getByPlaceholder('name@example.com').fill(user.email); + await page.getByPlaceholder('Your password').fill(user.password); + await page.getByRole('button', { name: 'Login' }).click(); + await page.waitForURL(new RegExp(REDIRECT_PATH)); +} + +// Monotonic counter so emails minted within the same millisecond by one worker +// never collide. +let userSeq = 0; + /** - * Extend the base test with an `authedPage` fixture that reuses a - * pre-authenticated storageState so we never re-login per test. + * Sign up a brand-new account via the Better Auth API in the given context and + * return its credentials. Sign-up auto-signs-in, so the context is left + * authenticated (its cookie jar holds the session). */ -export const test = base.extend<{ - authedPage: import('@playwright/test').Page; -}>({ - authedPage: async ({ browser }, use) => { - const context = await browser.newContext({ - storageState: STORAGE_STATE, - }); +export async function createFreshUser( + context: BrowserContext, + baseURL: string, + tag: string | number = 'x', +) { + const email = `e2e-${tag}-${Date.now()}-${userSeq++}@iridium.test`; + const password = 'password123'; + + const res = await context.request.post('/api/auth/sign-up/email', { + // Better Auth checks the request Origin against its trusted origins. + headers: { Origin: baseURL }, + data: { name: `E2E User ${tag}`, email, password }, + }); + + if (!res.ok()) { + throw new Error( + `Fresh-user sign-up failed (${res.status()}): ${await res.text()}`, + ); + } + + return { email, password }; +} + +/** + * Create a fresh browser context already authenticated as a brand-new user. + * Caller is responsible for closing the returned context. + */ +export async function createAuthedContext( + browser: Browser, + baseURL: string, + tag: string | number = 'x', +): Promise { + const context = await browser.newContext({ baseURL }); + await createFreshUser(context, baseURL, tag); + return context; +} + +/** + * Create a thread for the user owning `context` and return its id. Uses the + * /chat action (the same path the New Thread button hits) and reads the id + * from the redirect Location. + */ +export async function createThreadViaApi( + context: BrowserContext, +): Promise { + const res = await context.request.post('/chat', { + form: { intent: 'new-thread' }, + maxRedirects: 0, + }); + + const location = res.headers()['location']; + if (!location) { + throw new Error( + `Expected a redirect creating a thread, got ${res.status()}`, + ); + } + + return location.split('/').pop()!; +} + +/** + * Extend the base test with an `authedPage` fixture backed by a brand-new user + * created per test. Each test therefore starts from a clean slate (zero + * threads, zero notes), which keeps parallel runs free of shared-state races. + */ +export const test = base.extend<{ authedPage: Page }>({ + authedPage: async ({ browser }, use, testInfo) => { + const baseURL = testInfo.project.use.baseURL; + if (!baseURL) throw new Error('baseURL is not configured'); + + const context = await browser.newContext({ baseURL }); + await createFreshUser(context, baseURL, testInfo.workerIndex); + const page = await context.newPage(); await use(page); await context.close(); diff --git a/tests/global-setup.ts b/tests/global-setup.ts index 889efc8..4697506 100644 --- a/tests/global-setup.ts +++ b/tests/global-setup.ts @@ -2,15 +2,21 @@ import type { FullConfig } from '@playwright/test'; const TEST_USERS = [ { name: 'Alice', email: 'alice@iridium.dev', password: 'password123' }, + { name: 'Bob', email: 'bob@iridium.dev', password: 'password123' }, ]; export default async function globalSetup(config: FullConfig) { - const baseURL = config.projects[0]?.use?.baseURL ?? 'http://localhost:5173'; + const baseURL = config.projects[0]?.use?.baseURL ?? 'http://localhost:7778'; for (const user of TEST_USERS) { const res = await fetch(`${baseURL}/api/auth/sign-up/email`, { method: 'POST', - headers: { 'Content-Type': 'application/json' }, + headers: { + 'Content-Type': 'application/json', + // Better Auth validates the request Origin against its trusted + // origins; a server-to-server fetch sends none, so set it. + Origin: baseURL, + }, body: JSON.stringify(user), }); diff --git a/tests/healthcheck.spec.ts b/tests/healthcheck.spec.ts new file mode 100644 index 0000000..efc0de5 --- /dev/null +++ b/tests/healthcheck.spec.ts @@ -0,0 +1,12 @@ +import { test, expect } from './fixtures'; + +test.describe('Healthcheck', () => { + test('reports ok when both databases are reachable', async ({ + request, + }) => { + const res = await request.get('/healthcheck'); + expect(res.status()).toBe(200); + expect(res.headers()['content-type']).toContain('application/json'); + await expect(res.json()).resolves.toEqual({ status: 'ok' }); + }); +}); diff --git a/tests/home.spec.ts b/tests/home.spec.ts index e421e88..a1c4145 100644 --- a/tests/home.spec.ts +++ b/tests/home.spec.ts @@ -1,6 +1,26 @@ import { test, expect } from './fixtures'; -test('homepage loads', async ({ page }) => { - await page.goto('/'); - await expect(page).toHaveTitle(/Iridium/i); +test.describe('Home', () => { + test('loads with the expected title', async ({ page }) => { + await page.goto('/'); + await expect(page).toHaveTitle(/Iridium/i); + }); + + test('renders the marketing content', async ({ page }) => { + await page.goto('/'); + await expect( + page.getByRole('heading', { name: 'Iridium', level: 1 }), + ).toBeVisible(); + await expect( + page.getByRole('heading', { name: 'What is included' }), + ).toBeVisible(); + await expect( + page.getByRole('heading', { name: 'The stack' }), + ).toBeVisible(); + }); + + test('is reachable without authentication', async ({ page }) => { + await page.goto('/'); + await expect(page.getByRole('link', { name: 'Login' })).toBeVisible(); + }); }); diff --git a/tests/navigation.spec.ts b/tests/navigation.spec.ts new file mode 100644 index 0000000..e6b5d34 --- /dev/null +++ b/tests/navigation.spec.ts @@ -0,0 +1,117 @@ +import { test, expect } from './fixtures'; + +const siteNav = (page: import('@playwright/test').Page) => + page.getByRole('navigation', { name: 'Site' }); +const mainNav = (page: import('@playwright/test').Page) => + page.getByRole('navigation', { name: 'Main navigation' }); + +test.describe('Desktop navigation', () => { + test('logged-out: shows Login, hides Chat and Logout', async ({ page }) => { + await page.goto('/'); + + await expect( + siteNav(page).getByRole('link', { name: 'Login' }), + ).toBeVisible(); + await expect( + siteNav(page).getByRole('button', { name: 'Logout' }), + ).toHaveCount(0); + + await expect( + mainNav(page).getByRole('link', { name: 'Home' }), + ).toBeVisible(); + await expect( + mainNav(page).getByRole('link', { name: 'Chat' }), + ).toHaveCount(0); + }); + + test('logged-in: shows Logout and Chat, hides Login', async ({ + authedPage: page, + }) => { + await page.goto('/'); + + await expect( + siteNav(page).getByRole('button', { name: 'Logout' }), + ).toBeVisible(); + await expect( + siteNav(page).getByRole('link', { name: 'Login' }), + ).toHaveCount(0); + + await expect( + mainNav(page).getByRole('link', { name: 'Chat' }), + ).toBeVisible(); + }); + + test('brand link returns to home', async ({ authedPage: page }) => { + await page.goto('/chat'); + await siteNav(page).getByRole('link', { name: 'Iridium' }).click(); + await expect(page).toHaveURL(/\/$/); + await expect( + page.getByRole('heading', { name: 'Iridium', level: 1 }), + ).toBeVisible(); + }); + + test('footer is present', async ({ page }) => { + await page.goto('/'); + await expect( + page.getByText('Iridium. Go build. Be bold.'), + ).toBeVisible(); + }); + + test('skip-to-content link is keyboard reachable', async ({ page }) => { + await page.goto('/'); + const skip = page.getByRole('link', { name: 'Skip to main content' }); + await expect(skip).toHaveAttribute('href', '#main-content'); + + // Focusing reveals it (focus:not-sr-only) and activating jumps to main. + await skip.focus(); + await expect(skip).toBeVisible(); + await skip.press('Enter'); + await expect(page).toHaveURL(/#main-content$/); + }); +}); + +test.describe('Mobile navigation', () => { + test.use({ viewport: { width: 375, height: 720 } }); + + test('hamburger opens the drawer and Escape closes it', async ({ + authedPage: page, + }) => { + await page.goto('/'); + + const hamburger = page.getByRole('button', { + name: 'Open navigation menu', + }); + await expect(hamburger).toBeVisible(); + await expect(hamburger).toHaveAttribute('aria-expanded', 'false'); + + // Desktop site links are hidden at this width. + await expect( + siteNav(page).getByRole('button', { name: 'Logout' }), + ).toBeHidden(); + + await hamburger.click(); + await expect(hamburger).toHaveAttribute('aria-expanded', 'true'); + + const drawer = page.getByRole('navigation', { + name: 'Mobile navigation', + }); + await expect(drawer.getByRole('link', { name: 'Home' })).toBeVisible(); + await expect(drawer.getByRole('link', { name: 'Chat' })).toBeVisible(); + + await page.keyboard.press('Escape'); + await expect(hamburger).toHaveAttribute('aria-expanded', 'false'); + }); + + test('logged-out drawer shows Login', async ({ page }) => { + await page.goto('/'); + await page + .getByRole('button', { name: 'Open navigation menu' }) + .click(); + + const drawer = page.getByRole('navigation', { + name: 'Mobile navigation', + }); + await expect(drawer.getByRole('link', { name: 'Login' })).toBeVisible(); + await expect(drawer.getByRole('link', { name: 'Chat' })).toHaveCount(0); + }); +}); diff --git a/tests/security.spec.ts b/tests/security.spec.ts new file mode 100644 index 0000000..6e48e76 --- /dev/null +++ b/tests/security.spec.ts @@ -0,0 +1,53 @@ +import { + test, + expect, + createAuthedContext, + createThreadViaApi, +} from './fixtures'; + +test.describe('Thread access control', () => { + test('shows 404 for a thread that does not exist', async ({ + authedPage: page, + }) => { + await page.goto('/chat/this-thread-does-not-exist'); + const alert = page.getByRole('alert'); + await expect(alert).toBeVisible(); + await expect(alert).toContainText('404'); + }); + + test("cannot open another user's thread (403)", async ({ + authedPage: page, + browser, + baseURL, + }) => { + const owner = await createAuthedContext(browser, baseURL!, 'owner'); + try { + const threadId = await createThreadViaApi(owner); + + // The attacker is the fresh user behind authedPage. + await page.goto(`/chat/${threadId}`); + const alert = page.getByRole('alert'); + await expect(alert).toBeVisible(); + await expect(alert).toContainText('403'); + } finally { + await owner.close(); + } + }); + + test('redirects to login when opening a thread while logged out', async ({ + page, + browser, + baseURL, + }) => { + // Create a real thread as some user so the id is valid; the point is + // that an anonymous visitor never gets that far. + const owner = await createAuthedContext(browser, baseURL!, 'owner'); + try { + const threadId = await createThreadViaApi(owner); + await page.goto(`/chat/${threadId}`); + await expect(page).toHaveURL(/\/login/); + } finally { + await owner.close(); + } + }); +});