From 5ce280dd854aa6171db2efd351e2c36d7e4eb359 Mon Sep 17 00:00:00 2001 From: hshum Date: Fri, 17 Jul 2026 14:13:56 -0700 Subject: [PATCH] feat(qa): full-surface contract coverage, contract-fed vision judge, nightly loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Roadmap items 1-3 from the agent-native QA plan: 1. Coverage — the contract gains replay-miss (unknown /r/{hash} share links must degrade into the workspace with an honest banner) and mobile-entry-redirect states, with new runner vocabulary (per-state viewport, expectUrlPattern, expect/forbidTestId). Probing the live bundle showed the standalone mobile shell no longer mounts since the one-surface contraction, so live-smoke-mobile — red against current production while living outside CI, the exact #567 failure mode — is rewritten to assert the one-surface reality. 7/7 runner, 6/6 mobile smoke against the served bundle. 2. Vision judge grounding — the Gemini QA judge prompt now receives every contract clause as CONTRACT-VERIFIED INVARIANTS ground truth, so paid vision passes stop re-litigating what CI already proves and findings that contradict a verified clause classify as misreads. 3. Cadence — nightly-design-loop.yml builds main, executes every surface contract (fails on drift), captures the 4-variant screenshot set, and uploads both as reviewable artifacts. Co-Authored-By: Claude Fable 5 --- .github/workflows/nightly-design-loop.yml | 65 +++++++++++++ .../surfaces/decision-workspace.contract.json | 16 ++++ scripts/ui/runDogfoodGeminiQa.mjs | 35 ++++++- tests/e2e/live-smoke-mobile.spec.ts | 94 +++++++------------ tests/e2e/ui-contract-runner.spec.ts | 39 +++++++- 5 files changed, 184 insertions(+), 65 deletions(-) create mode 100644 .github/workflows/nightly-design-loop.yml diff --git a/.github/workflows/nightly-design-loop.yml b/.github/workflows/nightly-design-loop.yml new file mode 100644 index 000000000..b4c914a79 --- /dev/null +++ b/.github/workflows/nightly-design-loop.yml @@ -0,0 +1,65 @@ +# Nightly design loop — the hand-run capture → contract → review loop, on a +# cadence. Builds main, serves the production bundle, executes every surface +# contract (fails the run on drift), captures the 4-variant screenshot set, +# and uploads both as artifacts so any agent or human can review the visual +# state of the app each morning without rebuilding anything. +name: Nightly design loop + +on: + schedule: + - cron: "17 9 * * *" # 09:17 UTC ≈ 2:17am PT, after the day's merges settle + workflow_dispatch: {} + +env: + VITE_CONVEX_URL: ${{ secrets.CONVEX_URL || vars.CONVEX_URL || 'https://agile-caribou-964.convex.cloud' }} + +jobs: + design-loop: + name: Contract sweep + screenshot capture + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v6 + with: + package-manager-cache: false + node-version: 22 + + - name: Install dependencies + run: npm install --no-audit --no-fund + + - name: Install Chromium + run: npx playwright install --with-deps chromium + + - name: Build production bundle + run: npm run build + + - name: Serve bundle + run: | + npx vite preview --host 127.0.0.1 --port 4173 & + for i in $(seq 1 30); do + curl -sf http://127.0.0.1:4173/ > /dev/null && break + sleep 1 + done + + - name: Execute surface contracts (fails on drift) + env: + BASE_URL: http://127.0.0.1:4173 + run: npx playwright test tests/e2e/ui-contract-runner.spec.ts --project=chromium --workers=1 --reporter=line + + - name: Capture 4-variant screenshot set + env: + BASE_URL: http://127.0.0.1:4173 + DOGFOOD_SCREENSHOT_DIR: nightly-captures + run: npx playwright test tests/e2e/full-ui-dogfood.spec.ts --project=chromium --workers=1 --reporter=line + + - name: Upload screenshots + if: always() + uses: actions/upload-artifact@v7 + with: + name: nightly-design-loop-${{ github.run_id }} + path: | + nightly-captures/ + playwright-report/ + retention-days: 14 diff --git a/docs/design/ui-contract/surfaces/decision-workspace.contract.json b/docs/design/ui-contract/surfaces/decision-workspace.contract.json index 2ee0f6a7b..06a1eb6ee 100644 --- a/docs/design/ui-contract/surfaces/decision-workspace.contract.json +++ b/docs/design/ui-contract/surfaces/decision-workspace.contract.json @@ -41,6 +41,22 @@ "url": "/redesign/chat?report=__contract_missing__&artifact=notebook", "expectText": "Notebook context unavailable", "forbidText": "Notebook context selected" + }, + { + "name": "replay-miss", + "why": "an unknown /r/{hash} share link must degrade into the live workspace with an honest banner, never a blank page or a fabricated receipt", + "url": "/redesign/chat/r/__contract_missing__", + "expectText": "That reproducible answer is unavailable", + "expectTestId": "continued-chat-context" + }, + { + "name": "mobile-entry-redirect", + "why": "since the one-surface contraction (#561) every entry point resolves to the workspace; the legacy mobile shell must not resurface", + "url": "/", + "viewport": "mobile", + "expectUrlPattern": "/redesign/chat", + "expectTestId": "one-surface-workspace", + "forbidTestId": "mobile-home-surface" } ], "answerPacket": { diff --git a/scripts/ui/runDogfoodGeminiQa.mjs b/scripts/ui/runDogfoodGeminiQa.mjs index 7d6cc16dd..2b11752fa 100644 --- a/scripts/ui/runDogfoodGeminiQa.mjs +++ b/scripts/ui/runDogfoodGeminiQa.mjs @@ -1,4 +1,33 @@ import path from "node:path"; +import { readdirSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +/** + * Ground truth for the vision judge: every clause in the executable surface + * contracts (docs/design/ui-contract/surfaces/*.contract.json) is verified + * deterministically in CI by ui-contract-runner.spec.ts. A vision finding that + * contradicts a verified clause is a misread, not a bug — feeding the clause + * list to the judge stops paid vision passes from re-litigating what CI + * already proved (and cuts the score variance the Gemini loop is known for). + */ +function contractVerifiedInvariants() { + try { + const dir = path.join( + path.dirname(fileURLToPath(import.meta.url)), + "..", "..", "docs", "design", "ui-contract", "surfaces", + ); + const lines = []; + for (const file of readdirSync(dir).filter((f) => f.endsWith(".contract.json"))) { + const contract = JSON.parse(readFileSync(path.join(dir, file), "utf8")); + for (const anchor of contract.anchors ?? []) lines.push(`- [${contract.surface}] ${anchor.why}`); + for (const rule of contract.geometry ?? []) lines.push(`- [${contract.surface}] ${rule.why}`); + for (const state of contract.states ?? []) lines.push(`- [${contract.surface}] state "${state.name}": ${state.why}`); + } + return lines.join("\n"); + } catch { + return ""; + } +} import fs from "node:fs/promises"; import { existsSync, readFileSync } from "node:fs"; import net from "node:net"; @@ -2138,11 +2167,15 @@ async function judgeIssuesWithLLM(issues) { return `ISSUE #${i + 1} [${route}]: ${header}\n${details}`; }).join("\n\n---\n\n"); + const verifiedInvariants = contractVerifiedInvariants(); const prompt = `You are a senior UI/UX quality judge reviewing automated QA findings for a web application. DESIGN CONTEXT: ${DESIGN_CONTEXT} - +${verifiedInvariants ? ` +CONTRACT-VERIFIED INVARIANTS (machine-checked in CI on every PR — treat as ground truth; a finding that contradicts one of these is a screenshot misread, not a bug): +${verifiedInvariants} +` : ""} TASK: For each issue below, classify it into exactly ONE category: - "genuine_bug": A real, actionable defect that would affect users (broken layout, actual contrast failure, real typo, data corruption visible to end users) - "design_opinion": A subjective preference about intentional design choices (density, spacing, color hierarchy, information architecture, terminology for target audience) diff --git a/tests/e2e/live-smoke-mobile.spec.ts b/tests/e2e/live-smoke-mobile.spec.ts index f396688d2..66457ac8c 100644 --- a/tests/e2e/live-smoke-mobile.spec.ts +++ b/tests/e2e/live-smoke-mobile.spec.ts @@ -1,3 +1,18 @@ +/** + * live-smoke-mobile — Tier B / post-deploy mobile verification. + * + * Rewritten 2026-07-17: the previous version asserted the standalone mobile + * shell surfaces (mobile-home-surface, mobile-reports-surface, ...), which + * stopped mounting when #561 contracted NodeBench to the single decision + * workspace — every entry route now resolves to /redesign/chat at every + * viewport. The old assertions were red against current production while + * living outside CI: exactly the "test the pipeline cannot see" failure mode + * issue #567 documented. This spec now asserts the one-surface reality; + * richer per-clause coverage lives in the surface contract + * (docs/design/ui-contract/surfaces/decision-workspace.contract.json) executed + * by ui-contract-runner.spec.ts. + */ + import { expect, test } from "@playwright/test"; const BASE_URL = @@ -19,68 +34,27 @@ async function expectNoHorizontalOverflow(page: import("@playwright/test").Page) expect(fits, "mobile runtime surface should not overflow horizontally").toBe(true); } -test.describe("live-smoke-mobile — canonical runtime surfaces", () => { +test.describe("live-smoke-mobile — one workspace at 390px", () => { test.setTimeout(45_000); - test("Home keeps the mobile wrapper around the runtime-backed Home surface", async ({ page }) => { - await page.goto(`${BASE_URL}/`); - const surface = page.locator('[data-testid="mobile-home-surface"]'); - await expect(surface).toBeVisible({ timeout: 20_000 }); - await expect(surface.locator('[data-testid="exact-web-home-surface"]')).toBeVisible(); - await expect(surface.locator('[data-testid="exact-web-home-composer"]')).toBeVisible(); - await expect(surface.locator('[data-testid="mobile-home-async-run"]')).toHaveCount(0); - await expectNoHorizontalOverflow(page); - }); - - test("Reports exposes real pipeline controls and honest runtime report state", async ({ page }) => { - await page.goto(`${BASE_URL}/?surface=reports`); - const surface = page.locator('[data-testid="mobile-reports-surface"]'); - await expect(surface).toBeVisible({ timeout: 20_000 }); - await expect(surface.locator('[data-testid="reports-performance-root"]')).toBeVisible(); - await expect(surface.locator('[data-testid="pipeline-launcher"]')).toBeVisible(); - await expect(surface.locator('[data-testid="pipeline-launcher-sign-in"]')).toBeVisible(); - await expect( - surface.locator('[data-testid="reports-loading"], [data-testid="reports-empty"], [data-testid="report-card"]').first(), - ).toBeVisible(); - await expect(surface.locator('[data-testid="mobile-report-card"]')).toHaveCount(0); - await expectNoHorizontalOverflow(page); - }); - - test("Chat mounts the live runtime and a working composer, not a verified fixture", async ({ page }) => { - await page.goto(`${BASE_URL}/?surface=chat`); - const surface = page.locator('[data-testid="mobile-chat-surface"]'); - await expect(surface).toBeVisible({ timeout: 20_000 }); - await expect(surface.locator('[data-testid="exact-web-chat-stream"]')).toBeVisible(); - await expect(surface.getByLabel("Chat composer")).toBeVisible(); - await expect(surface.getByLabel("Chat composer")).toHaveAttribute( - "placeholder", - "Ask or paste text... (@ to mention an entity)", - ); - await expect(surface.getByText("Live Context Runtime")).toBeVisible(); - await expect(surface.getByText(/3 entities - 24 sources/i)).toHaveCount(0); - await expectNoHorizontalOverflow(page); - }); - - test("Inbox shows only the runtime snapshot or its honest empty/loading state", async ({ page }) => { - await page.goto(`${BASE_URL}/?surface=inbox`); - const surface = page.locator('[data-testid="mobile-inbox-surface"]'); - await expect(surface).toBeVisible({ timeout: 20_000 }); - await expect(surface.locator('[data-testid="exact-web-inbox-surface"]')).toBeVisible(); - await expect(surface.getByRole("button", { name: /^All / })).toBeVisible(); - await expect(surface.getByRole("button", { name: /^Mentions / })).toHaveCount(0); - await expectNoHorizontalOverflow(page); - }); - - test("Me uses runtime identity and memory rather than canned workspace controls", async ({ page }) => { - await page.goto(`${BASE_URL}/?surface=me`); - const surface = page.locator('[data-testid="mobile-me-surface"]'); - await expect(surface).toBeVisible({ timeout: 20_000 }); - await expect(surface.locator('[data-testid="exact-web-me-surface"]')).toBeVisible(); - await expect(surface.getByRole("button", { name: "Settings" })).toBeVisible(); - await expect( - surface.locator('[data-testid="me-memory-loading"], [data-testid="me-memory-empty"], .nb-settings-section').first(), - ).toBeVisible(); - await expect(surface.getByText(/^Workspaces$/)).toHaveCount(0); + for (const path of ["/", "/?surface=reports", "/?surface=chat", "/?surface=inbox", "/?surface=me"]) { + test(`${path} resolves to the single decision workspace`, async ({ page }) => { + await page.goto(`${BASE_URL}${path}`); + await expect(page).toHaveURL(/\/redesign\/chat(?:[/?#]|$)/, { timeout: 20_000 }); + await expect(page.getByTestId("one-surface-workspace")).toBeVisible({ timeout: 20_000 }); + // Exactly one composer; the legacy mobile shell must not resurface. + await expect(page.locator("textarea:visible")).toHaveCount(1); + await expect(page.getByTestId("mobile-home-surface")).toHaveCount(0); + await expectNoHorizontalOverflow(page); + }); + } + + test("unknown share link degrades honestly on mobile", async ({ page }) => { + await page.goto(`${BASE_URL}/redesign/chat/r/__smoke_missing__`); + await expect(page.getByText("That reproducible answer is unavailable").first()).toBeVisible({ + timeout: 20_000, + }); + await expect(page.getByTestId("one-surface-workspace")).toBeVisible({ timeout: 20_000 }); await expectNoHorizontalOverflow(page); }); }); diff --git a/tests/e2e/ui-contract-runner.spec.ts b/tests/e2e/ui-contract-runner.spec.ts index aba282adc..f4152b13e 100644 --- a/tests/e2e/ui-contract-runner.spec.ts +++ b/tests/e2e/ui-contract-runner.spec.ts @@ -52,7 +52,17 @@ interface SurfaceContract { gridTracks?: number; minWidthPx?: number; }>; - states?: Array<{ name: string; why: string; url: string; expectText: string; forbidText?: string }>; + states?: Array<{ + name: string; + why: string; + url: string; + viewport?: string; + expectText?: string; + forbidText?: string; + expectTestId?: string; + forbidTestId?: string; + expectUrlPattern?: string; + }>; } function loadContracts(): SurfaceContract[] { @@ -130,14 +140,35 @@ for (const contract of contracts) { for (const state of contract.states ?? []) { test(`state › ${state.name}: copy agrees with reality`, async ({ page }) => { + const viewport = state.viewport + ? contract.viewports.find((v) => v.name === state.viewport) + : undefined; + if (viewport) { + await page.setViewportSize({ width: viewport.width, height: viewport.height }); + } await installVercelPreviewBypass(page, BASE_URL); await page.goto(`${BASE_URL}${state.url}`, { waitUntil: "domcontentloaded" }); - await expect(page.getByText(state.expectText).first(), state.why).toBeVisible({ - timeout: 30_000, - }); + if (state.expectUrlPattern) { + await expect(page, state.why).toHaveURL(new RegExp(state.expectUrlPattern), { + timeout: 30_000, + }); + } + if (state.expectText) { + await expect(page.getByText(state.expectText).first(), state.why).toBeVisible({ + timeout: 30_000, + }); + } + if (state.expectTestId) { + await expect(page.getByTestId(state.expectTestId).first(), state.why).toBeVisible({ + timeout: 30_000, + }); + } if (state.forbidText) { await expect(page.getByText(state.forbidText), state.why).toHaveCount(0); } + if (state.forbidTestId) { + await expect(page.getByTestId(state.forbidTestId), state.why).toHaveCount(0); + } }); } });