From 536285f35c9c35dc29281f34be1e77124659a550 Mon Sep 17 00:00:00 2001 From: Duncan McKinnon Date: Wed, 12 Aug 2026 11:32:30 -0700 Subject: [PATCH 1/2] adding docs section --- README.md | 11 ++- web/src/App.test.tsx | 46 ++++++++++ web/src/App.tsx | 5 ++ web/src/components/Layout.test.tsx | 37 +++++++- web/src/components/Layout.tsx | 10 ++- web/src/components/icons.test.tsx | 2 + web/src/components/icons.tsx | 12 +++ web/src/docs/content/Cli.tsx | 90 +++++++++++++++++++ web/src/docs/content/Datasets.tsx | 98 +++++++++++++++++++++ web/src/docs/content/Evals.tsx | 96 ++++++++++++++++++++ web/src/docs/content/Keys.tsx | 137 +++++++++++++++++++++++++++++ web/src/docs/content/Runs.tsx | 87 ++++++++++++++++++ web/src/docs/primitives.test.tsx | 112 +++++++++++++++++++++++ web/src/docs/primitives.tsx | 88 ++++++++++++++++++ web/src/docs/registry.test.tsx | 75 ++++++++++++++++ web/src/docs/registry.ts | 40 +++++++++ web/src/pages/DocsPage.test.tsx | 114 ++++++++++++++++++++++++ web/src/pages/DocsPage.tsx | 52 +++++++++++ web/src/styles.css | 126 ++++++++++++++++++++++++++ 19 files changed, 1233 insertions(+), 5 deletions(-) create mode 100644 web/src/App.test.tsx create mode 100644 web/src/docs/content/Cli.tsx create mode 100644 web/src/docs/content/Datasets.tsx create mode 100644 web/src/docs/content/Evals.tsx create mode 100644 web/src/docs/content/Keys.tsx create mode 100644 web/src/docs/content/Runs.tsx create mode 100644 web/src/docs/primitives.test.tsx create mode 100644 web/src/docs/primitives.tsx create mode 100644 web/src/docs/registry.test.tsx create mode 100644 web/src/docs/registry.ts create mode 100644 web/src/pages/DocsPage.test.tsx create mode 100644 web/src/pages/DocsPage.tsx diff --git a/README.md b/README.md index debd40c..2878a5e 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Either way you get an `valcore` command on your `PATH`. ## Web UI -`valcore serve` opens a dark-themed web UI with four surfaces: +`valcore serve` opens a dark-themed web UI with five surfaces: - **Overview** — the landing page, summarizing what you have and pointing to the next step. @@ -65,6 +65,12 @@ Either way you get an `valcore` command on your `PATH`. from a description. - **Runs** — inspect completed runs, their metrics, and per-row scores, and compare runs against each other. +- **Docs** — how the product works, in four tabs: Evals, Datasets, Runs, and CLI. + +The docs surface explains the concepts the other four assume — label spaces, frozen +versions, run kinds, agreement metrics — so you can read them without leaving the app. +It covers the workflow; this README stays the reference for install, credentials, +portable packages, CI, and Logfire. ## Quickstart @@ -80,6 +86,9 @@ valcore serve and datasets in the UI, then drive runs from the command line. Both can be written by hand or generated from a description; a generated result is an editable draft either way. +From there, the **Docs** tab in the app walks through the full workflow — authoring a +judge, getting labeled rows, and reading what a run measured. + ## Seeding one from the other An evaluator and a dataset have to agree on columns, so rather than retype that shape you diff --git a/web/src/App.test.tsx b/web/src/App.test.tsx new file mode 100644 index 0000000..9f1b4b5 --- /dev/null +++ b/web/src/App.test.tsx @@ -0,0 +1,46 @@ +// Route-table coverage. DocsPage.test.tsx mounts the docs routes itself, which proves the +// component but not that App declares them — this file is what fails if /docs is missing +// from the real route table. +// +// Only docs paths are exercised: the other pages fetch on mount, and this suite is about +// wiring, not about mocking every endpoint. +import { afterEach, describe, expect, it } from "vitest"; +import { cleanup, render, screen } from "@testing-library/react"; +import { MemoryRouter } from "react-router-dom"; +import App from "./App"; +import { DOCS } from "./docs/registry"; + +afterEach(() => { + cleanup(); +}); + +function renderApp(path: string) { + return render( + + + , + ); +} + +describe("App routes", () => { + it("serves the docs section at /docs", () => { + renderApp("/docs"); + + expect(screen.getByRole("heading", { level: 1 }).textContent).toBe(DOCS[0].title); + }); + + it("serves a docs tab at /docs/:slug", () => { + renderApp("/docs/runs"); + + expect(screen.getByRole("heading", { level: 1 }).textContent).toBe("Runs"); + }); + + it("renders the docs section inside the app shell", () => { + renderApp("/docs"); + + // Two navs: the sidebar and the tab strip. If docs were declared outside the layout + // route, the sidebar would vanish while reading them. + expect(screen.getAllByRole("navigation")).toHaveLength(2); + expect(screen.getByRole("link", { name: "Docs" }).getAttribute("href")).toBe("/docs"); + }); +}); diff --git a/web/src/App.tsx b/web/src/App.tsx index fd1e909..395ebee 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -4,6 +4,7 @@ import OverviewPage from "./pages/OverviewPage"; import EvaluatorsPage from "./pages/EvaluatorsPage"; import DatasetsPage from "./pages/DatasetsPage"; import RunsPage from "./pages/RunsPage"; +import DocsPage from "./pages/DocsPage"; // All routes are declared here up front. The route table is maintained alongside the // pages, so adding or moving a route means editing this file. @@ -19,6 +20,10 @@ export default function App() { } /> } /> } /> + {/* Both point at DocsPage: the bare path renders the first tab, so the section + always has content and a stale slug degrades instead of blanking. */} + } /> + } /> ); diff --git a/web/src/components/Layout.test.tsx b/web/src/components/Layout.test.tsx index 55758ac..53cda27 100644 --- a/web/src/components/Layout.test.tsx +++ b/web/src/components/Layout.test.tsx @@ -33,14 +33,47 @@ describe("Layout nav", () => { expect(screen.getByRole("link", { name: "Datasets" })).toBeTruthy(); expect(screen.getByRole("link", { name: "Runs" })).toBeTruthy(); expect(screen.getByRole("link", { name: "Compare" })).toBeTruthy(); + expect(screen.getByRole("link", { name: "Docs" })).toBeTruthy(); }); - it("renders exactly the five expected nav links and nothing else", () => { + it("renders exactly the six expected nav links and nothing else", () => { renderLayout("/"); // Guards against a stray link (e.g. the brand wordmark accidentally becoming // a link, or a footer/version badge that this task must not add). - expect(screen.getAllByRole("link")).toHaveLength(5); + expect(screen.getAllByRole("link")).toHaveLength(6); + }); + + it("orders Docs directly under Overview, above the labelled groups", () => { + renderLayout("/"); + + // Docs sits with Overview as the two ungrouped entries at the top: it is read + // before you have anything to author or measure, so it should not be buried under + // the working surfaces. + expect(screen.getAllByRole("link").map((link) => link.textContent)).toEqual([ + "Overview", + "Docs", + "Evaluators", + "Datasets", + "Runs", + "Compare", + ]); + }); + + it("points Docs at /docs", () => { + renderLayout("/"); + + expect(screen.getByRole("link", { name: "Docs" }).getAttribute("href")).toBe("/docs"); + }); + + it("keeps Docs active on a docs sub-route", () => { + renderLayout("/docs/datasets"); + + // Docs deliberately omits `end`: every /docs/:slug tab must keep the nav item + // lit, otherwise the sidebar goes blank-looking while reading any tab but the + // first. + expect(screen.getByRole("link", { name: "Docs" }).getAttribute("aria-current")).toBe("page"); + expect(screen.getByRole("link", { name: "Overview" }).getAttribute("aria-current")).toBeNull(); }); it("points Overview at / and Compare at /runs/compare", () => { diff --git a/web/src/components/Layout.tsx b/web/src/components/Layout.tsx index 9529b09..1538f1a 100644 --- a/web/src/components/Layout.tsx +++ b/web/src/components/Layout.tsx @@ -3,6 +3,7 @@ import { NavLink, Outlet } from "react-router-dom"; import { CompareIcon, DatasetIcon, + DocsIcon, EvaluatorIcon, OverviewIcon, RunIcon, @@ -11,9 +12,12 @@ import { type NavItem = { to: string; label: string; Icon: ComponentType<{ className?: string }> }; type NavSection = { label: string; items: NavItem[] }; -// Overview stands alone above the labelled groups; the two sections mirror the -// author-then-measure flow of the product. +// Overview and Docs are the two ungrouped entries at the top; the labelled sections +// below them mirror the author-then-measure flow of the product. Docs sits up here +// rather than at the bottom because it is what you read before you have anything to +// author or measure — a new user needs it first, not last. const OVERVIEW: NavItem = { to: "/", label: "Overview", Icon: OverviewIcon }; +const DOCS: NavItem = { to: "/docs", label: "Docs", Icon: DocsIcon }; const SECTIONS: NavSection[] = [ { label: "Author", @@ -56,6 +60,8 @@ export default function Layout() { {/* `end` keeps the "/" link from matching every route and staying active. */} + {/* No `end`: every /docs/:slug tab keeps the nav item lit. */} + {SECTIONS.map((section) => (
{section.label}
diff --git a/web/src/components/icons.test.tsx b/web/src/components/icons.test.tsx index de68d0c..dc4c867 100644 --- a/web/src/components/icons.test.tsx +++ b/web/src/components/icons.test.tsx @@ -4,6 +4,7 @@ import { ChevronIcon, CompareIcon, DatasetIcon, + DocsIcon, EvaluatorIcon, InfoIcon, OverviewIcon, @@ -26,6 +27,7 @@ const ICONS = [ ["InfoIcon", InfoIcon], ["ChevronIcon", ChevronIcon], ["PlusIcon", PlusIcon], + ["DocsIcon", DocsIcon], ] as const; describe("icons", () => { diff --git a/web/src/components/icons.tsx b/web/src/components/icons.tsx index 3f019ac..eda3250 100644 --- a/web/src/components/icons.tsx +++ b/web/src/components/icons.tsx @@ -107,3 +107,15 @@ export function PlusIcon(props: IconProps): JSX.Element { ); } + +// An open book: two facing pages over a spine. Distinct from DatasetIcon's stacked +// rules at 16px, which matters because both sit in the same nav column. +export function DocsIcon(props: IconProps): JSX.Element { + return ( + + + + + + ); +} diff --git a/web/src/docs/content/Cli.tsx b/web/src/docs/content/Cli.tsx new file mode 100644 index 0000000..57cfe57 --- /dev/null +++ b/web/src/docs/content/Cli.tsx @@ -0,0 +1,90 @@ +// The CLI tab. Deliberately not a mirror of the README's command table: the README stays +// canonical for the exhaustive listing (it is the PyPI long_description, where a reader +// cannot reach these pages), and this tab teaches the usage patterns that table does not. + +import { CodeBlock, DocLink, DocNote, DocPage, DocSection } from "../primitives"; + +export function Cli(): JSX.Element { + return ( + + +

+ The app and the CLI are two front ends over the same SQLite workspace. Authoring is + easier in the app; running, exporting, and anything scripted is easier from the command + line. Nothing has to be running for the CLI to work — it opens the database directly, so + runs do not depend on valcore serve. +

+ valcore serve + + + +

+ Evaluators, versions, and datasets are addressable by name or by a unique id prefix, so + you rarely need a full id: +

+ valcore run 7f3a my-dataset +

+ An ambiguous value is an error that lists the candidates rather than picking one. Add + characters until it resolves. +

+ + + +
    +
  • + --version — a version other than the active one +
    • +
  • + --kind validation — compare against labels instead of only recording + output +
    • +
  • + --concurrency — how many rows are in flight at once +
    • +
  • + --watch — one line per completed row, rather than a final summary +
    • +
  • + --json — machine-readable results on stdout +
    • +
  • + --min-accuracy — exit non-zero below a threshold, for CI +
    • +
+ + + +

+ valcore config holds credentials and defaults — the gateway key, Logfire + tokens, the default model, and the path to the config file. Keys are only ever set from + the CLI; no secret crosses HTTP into the browser. +

+ valcore config set-key +

+ valcore list shows what the workspace holds, as a table or with{" "} + --json. valcore export and valcore import move + evaluators and datasets between machines. valcore logfire push sends a + dataset to Logfire's hosted store. valcore skills installs the bundled + agent skills so a coding agent can drive valcore for you. +

+ + + +

+ Every command takes --db on the group, which is how you keep a scratch + workspace or a per-project database separate from the default under{" "} + ~/.valcore: +

+ valcore --db ./evals.sqlite list evaluators + + For the complete command table, plus install, CI recipes, portable packages, and + Logfire setup, see the README in the repository. This tab covers the patterns, not + every flag. + +

+ Back to Runs for what the run commands measure. +

+ + + ); +} diff --git a/web/src/docs/content/Datasets.tsx b/web/src/docs/content/Datasets.tsx new file mode 100644 index 0000000..928432d --- /dev/null +++ b/web/src/docs/content/Datasets.tsx @@ -0,0 +1,98 @@ +// The datasets tab: where rows come from, how labels get onto them, and why labels are +// optional until you want to measure agreement. + +import { CodeBlock, DocLink, DocNote, DocPage, DocSection } from "../primitives"; + +export function Datasets(): JSX.Element { + return ( + + +

+ A dataset is a table of rows with named columns, and it arrives one of three ways. + Upload a CSV when you already have data. Create a blank dataset and author rows by + hand when you are working from a handful of known cases. Generate one from a + description when you need coverage you do not have yet. +

+

+ Generation is a starting point, not an answer: a generated dataset is an editable draft + like any other, and its rows are yours to correct. +

+ + + +

+ A generated dataset keeps the request that produced it — the description, the columns + asked for, and the per-column notes — as read-only provenance. Months later that is how + you know what these rows were meant to represent. Uploaded and blank datasets have no + such record, which is normal rather than missing. +

+

+ Prescribing a label mix is opt-in. Leave it off and the distribution follows whatever + the description asks for. Turn it on to pin the proportions — useful when you need a + rare category represented well enough to measure. The editor works in whole percents + and shows the apportioned row count beside each label, so the number the model is + actually told is never hidden behind a percentage. +

+ + + +

+ Labels are ground truth: what the right answer is, independent of what any judge says. + The labeling grid is built for getting through rows quickly with the keyboard. +

+
    +
  • + j / k — move between rows +
    • +
  • + 1–9 — apply a categorical label +
    • +
  • + a — accept the suggested label +
    • +
  • + u — clear the label +
    • +
  • + ? — show the full shortcut list +
    • +
+

+ Every change saves immediately, applied optimistically and rolled back if the write + fails. Cells are editable in place, rows can be added or deleted, and per-column notes + record what a column is supposed to hold. +

+ + + +

+ A dataset needs no labels to be scored. An ordinary run just records what the judge + said. Labels are only required for a validation run, which compares the judge against + them to measure agreement. +

+ + Validation is unavailable — and says so — while a dataset still has unlabeled rows. A + partially labeled dataset would produce an agreement number over a silently shrinking + subset. + + + + +

+ A dataset that turned out too small can be extended in place: generating more rows + reuses the original request so the additions match the shape and intent of what is + already there. +

+

+ Export writes a dataset out as a Python script or as a portable JSON package, and the + CLI reads it back: +

+ valcore export my-evaluator --dataset my-dataset --format json +

+ Work with data in Datasets, or see{" "} + Evals for the judge side of the column contract. +

+ + + ); +} diff --git a/web/src/docs/content/Evals.tsx b/web/src/docs/content/Evals.tsx new file mode 100644 index 0000000..0cb6ddd --- /dev/null +++ b/web/src/docs/content/Evals.tsx @@ -0,0 +1,96 @@ +// The evaluators tab. Written for someone with the version editor open in another tab, +// so it explains the concepts the form assumes rather than restating its field labels. + +import { CodeBlock, DocLink, DocNote, DocPage, DocSection } from "../primitives"; + +export function Evals(): JSX.Element { + return ( + + +

+ An evaluator is an LLM-as-judge: a prompt, a model, and an output shape that together + score one thing about a row of data. It reads the columns you give it and returns a + label — a category from a fixed set, or a number — plus its reasoning. +

+

+ An evaluator is not tied to a dataset. The same judge runs over any dataset that + supplies the columns it requires, which is what makes a judge comparable across data. +

+ + + +

+ The output field defines what a judge returns. Categorical fields declare their label + space up front — the set of allowed values — and that space is what the labeling grid + offers and what agreement metrics are computed over. Numeric fields declare a range + instead, and are scored with error measures rather than a confusion matrix. +

+

+ Declaring the label space is what lets valcore tell a disagreement from an invalid + answer. A judge that returns something outside the space is a failure, not a low score. +

+ + + +

+ Every change to a judge lands as a version, and one version is active — the one used + when you do not name another. Editing an unsaved draft edits in place. Editing a frozen + version copies it first, so a version that has already produced runs is never mutated + underneath them. +

+

+ That is the point of freezing: a run records which version produced it, so a score + stays attributable. When two versions exist you can read the exact difference between + them in the version diff rather than guessing from timestamps. +

+ + Validation runs every render and gates Save before the server sees the payload, so an + incomplete version tells you what is missing instead of failing on submit. + + + + +

+ A judge can be granted capabilities when reading the row is not enough. FileSystem + gives it a rooted directory to read from; Shell gives it an allow-listed set of + commands and a timeout. Both are opt-in per version and configured alongside the + prompt. +

+

+ Grant the narrowest thing that answers the question. A capability widens what a judge + can see, which also widens what can change between runs. +

+ + + +

+ An evaluator and a dataset have to agree on columns, so rather than retype that shape, + generate one from the other. A dataset generated from a version always gets that + version's required columns; you can name extra columns, and per-column notes say + what each should contain. Suggested labels are optional — ask for them when you want + the model to propose ground truth, and the label space comes from the evaluator. +

+

+ The reverse direction works too: generate an evaluator from a dataset and it is + drafted against that dataset's columns. The result is an editable draft, not a + saved version. +

+

+ Author judges in Evaluators, then see{" "} + Datasets for what happens to the data side. +

+ + + +

+ Runs are driven from the command line, against the active version unless you name + another: +

+ valcore run my-evaluator my-dataset +

+ See Runs for run kinds and what gets measured. +

+ + + ); +} diff --git a/web/src/docs/content/Keys.tsx b/web/src/docs/content/Keys.tsx new file mode 100644 index 0000000..1a1d780 --- /dev/null +++ b/web/src/docs/content/Keys.tsx @@ -0,0 +1,137 @@ +// The keys tab: the three credentials the Overview setup card lists, where each one comes +// from, and what stops working without it. Deliberately the first tab — nothing that calls +// a model runs until the gateway key exists. +// +// Key names, commands, and required/optional status here must match +// src/valcore/api/routes/setup.py, which is what the Overview card renders from. + +import { CodeBlock, DocLink, DocNote, DocPage, DocSection, ExternalLink } from "../primitives"; + +export function Keys(): JSX.Element { + return ( + + +

+ The setup card on Overview lists every credential valcore + knows about and whether it is currently set. One is required; two are optional and + only matter if you use Logfire. +

+
    +
  • + Pydantic AI Gateway key — required. Runs evaluators, and generates + evaluators and datasets. +
    • +
  • + Logfire write token — optional. Sends run traces to Logfire. +
    • +
  • + Logfire API key — optional. Pushes datasets to Logfire's hosted + store. +
    • +
+ + Keys are never entered through the web UI — no secret crosses HTTP — so every one of + them is set from the command line. The card reports presence only; it never shows a + key back to you. + + + + +

+ valcore reaches models through the Pydantic AI Gateway, and that is currently the only + route — there is no direct-to-provider client and no per-provider key. Without it, + generation and runs are unavailable and the UI says why. +

+

+ Everything that does not call a model still works with no key at all: authoring by + hand, uploading a CSV, editing rows, hand-labeling, and every export. +

+ valcore config set-key +

+ Run it with no argument to be prompted without the key echoing to your terminal or + landing in shell history. +

+ + + +

+ Create the key in the Pydantic AI Gateway and paste it into the command above. The + gateway documentation covers account setup and where keys are issued: +

+

+ + ai.pydantic.dev/gateway + +

+

+ Model strings are always gateway/<provider>:<model> — for + example gateway/anthropic:claude-sonnet-5, which is the default. Valid + providers are anthropic, openai, google,{" "} + google-cloud, bedrock, and groq. A string that + does not match this shape is rejected before any request is made, so a bare model name + fails immediately with a clear error rather than at call time. +

+ + + +

+ With a write token configured, each run opens a valcore.run span carrying + the evaluator version, dataset, and concurrency, with one child span per scored row. + On close, the run span records its status and each agreement metric as attributes, so + a Logfire query can filter runs by accuracy directly. +

+ valcore config set-logfire-token +

+ A write token belongs to one Logfire project and is created from that project's + settings. Create the project first, then issue the token there: +

+

+ + Creating write tokens + +

+ + Tracing is an optional extra as well as an optional key — install it with{" "} + uv tool install 'valcore[logfire]'. The gateway already reports + the LLM calls themselves; valcore adds only the surrounding run and row context, and + deliberately does not re-report the calls, which would double-count tokens and cost. + + + + +

+ The API key is a separate credential from the write token, and it is only needed for + one thing: pushing a dataset to Logfire's hosted dataset store. +

+ valcore config set-logfire-key +

+ It must carry the project:read_datasets and{" "} + project:write_datasets scopes. A key without them will authenticate and + then fail on the push. API keys are issued from your Logfire account settings: +

+

+ + Logfire API reference + +

+ valcore logfire push my-dataset + + + +

+ Keys live in ~/.valcore/config.toml, written with mode 0600, + and are exported into the environment when a command runs. An already-exported + environment variable always wins over the stored value, which is what you want in CI: +

+ export PYDANTIC_AI_GATEWAY_API_KEY=sk-... +

To check what is currently configured without revealing anything:

+ valcore config get +

+ The stored key is masked unless you pass --show-key. See{" "} + CLI for the rest of the config group, or{" "} + Evals to start authoring now that setup is done. +

+ + + ); +} diff --git a/web/src/docs/content/Runs.tsx b/web/src/docs/content/Runs.tsx new file mode 100644 index 0000000..abbf34c --- /dev/null +++ b/web/src/docs/content/Runs.tsx @@ -0,0 +1,87 @@ +// The runs tab: kinds, what each metric means, comparison, and the two commands that +// produce runs. Named Runs rather than Experiments because the UI says Runs and +// `valcore experiment` is one specific command rather than the whole idea. + +import { CodeBlock, DocLink, DocNote, DocPage, DocSection } from "../primitives"; + +export function Runs(): JSX.Element { + return ( + + +

+ A run is one evaluator version scored over one dataset, recorded with its results. The + version is part of the record, so a score stays attributable after the judge moves on. +

+

Runs are started from the command line or from the launcher in the app:

+ valcore run my-evaluator my-dataset + + + +

+ An ordinary run records what the judge returned for each row. That is enough to inspect + behavior, spot invalid outputs, and compare two judges against each other. +

+

+ A validation run additionally compares the judge against the dataset's labels and + reports agreement. It requires a fully labeled dataset, so the option is unavailable — + with the reason shown — when rows are still unlabeled. +

+ valcore run my-evaluator my-dataset --kind validation + + + +

+ What you get depends on the output field. A categorical judge produces a confusion + matrix: rows are the label, columns are what the judge said, and the diagonal is + agreement. Reading off the diagonal tells you which categories a judge confuses, which + a single accuracy number hides. +

+

+ A numeric judge produces error measures instead — mean absolute error and root mean + square error. MAE is the typical miss; RMSE punishes large misses harder, so a gap + between them means a few rows are badly wrong rather than everything being slightly + off. +

+ + + +

+ Comparison puts two runs over the same dataset side by side, disagreements first, + because the rows where they differ are the only ones that explain a change. Comparing + runs over different datasets is refused rather than rendered — the numbers would not be + about the same thing. +

+

+ Open Compare to pick two, or{" "} + Runs to inspect one on its own. +

+ + + +

+ valcore run uses valcore's own runner. valcore experiment{" "} + scores the same pairing through pydantic_evals.Dataset.evaluate instead, + which is the path to take when you want results inside the pydantic-evals ecosystem. +

+ valcore experiment my-evaluator my-dataset + + + +

+ A validation run can fail on its own accuracy, which is what makes it usable as a + check rather than a report: +

+ + valcore run my-evaluator my-dataset --kind validation --min-accuracy 0.9 + + + Progress goes to stderr and results to stdout, so redirecting stdout yields clean JSON. + The CLI talks to SQLite directly — runs work whether or not serve is up. + +

+ See CLI for the flags that shape a run. +

+ + + ); +} diff --git a/web/src/docs/primitives.test.tsx b/web/src/docs/primitives.test.tsx new file mode 100644 index 0000000..b8fc0a4 --- /dev/null +++ b/web/src/docs/primitives.test.tsx @@ -0,0 +1,112 @@ +// The four primitives every content file is built from. Content prose is covered by +// the registry smoke render; what needs real assertions is the behavior these carry: +// a Copy button that puts the exact command on the clipboard, and links that resolve +// to real in-app routes rather than dead anchors. +import { afterEach, describe, expect, it, vi } from "vitest"; +import { cleanup, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { MemoryRouter } from "react-router-dom"; +import { + CodeBlock, + DocLink, + DocNote, + DocPage, + DocSection, + ExternalLink, +} from "./primitives"; + +afterEach(() => { + cleanup(); +}); + +describe("DocPage", () => { + it("renders its children", () => { + render(body text); + + expect(screen.getByText("body text")).toBeTruthy(); + }); +}); + +describe("DocSection", () => { + it("titles the block with an h2, leaving h1 to the page header", () => { + render(how versions freeze); + + // Level 2, not 1: DocsPage owns the single

via PageHeader, so a section + // heading that claimed h1 would give the page two. + expect(screen.getByRole("heading", { level: 2, name: "Versions" })).toBeTruthy(); + expect(screen.getByText("how versions freeze")).toBeTruthy(); + }); +}); + +describe("CodeBlock", () => { + it("shows the command text", () => { + render(valcore run my-eval my-dataset); + + expect(screen.getByText("valcore run my-eval my-dataset")).toBeTruthy(); + }); + + it("copies the command to the clipboard", async () => { + const user = userEvent.setup(); + + // userEvent.setup() installs its own clipboard stub, so the spy must be installed + // after it runs, matching ExportModal.test.tsx's copy-testing convention. + const writeText = vi.fn(() => Promise.resolve()); + Object.defineProperty(navigator, "clipboard", { + value: { writeText }, + configurable: true, + writable: true, + }); + + render(valcore serve); + await user.click(screen.getByRole("button", { name: "Copy" })); + + // The exact string, not a trimmed or re-indented variant: a command that does not + // paste verbatim is worse than no Copy button. + expect(writeText).toHaveBeenCalledWith("valcore serve"); + }); +}); + +describe("DocNote", () => { + it("renders its children and hides the icon from assistive tech", () => { + const { container } = render(labels are only needed for validation); + + expect(screen.getByText("labels are only needed for validation")).toBeTruthy(); + expect(container.querySelector("svg")?.getAttribute("aria-hidden")).toBe("true"); + }); +}); + +describe("ExternalLink", () => { + it("renders a real anchor to the destination", () => { + // Not a router Link: react-router would treat the URL as an in-app path and route + // to a 404 shell instead of leaving the app. + render(gateway docs); + + expect(screen.getByRole("link", { name: "gateway docs" }).getAttribute("href")).toBe( + "https://ai.pydantic.dev/gateway/", + ); + }); + + it("opens in a new tab without leaking the opener", () => { + render(logfire); + + const link = screen.getByRole("link", { name: "logfire" }); + // The app is a local workspace with unsaved editor state; navigating away in the + // same tab would discard it. + expect(link.getAttribute("target")).toBe("_blank"); + expect(link.getAttribute("rel")).toBe("noreferrer"); + }); +}); + +describe("DocLink", () => { + it("links to an in-app route", () => { + render( + + Evaluators + , + ); + + expect(screen.getByRole("link", { name: "Evaluators" }).getAttribute("href")).toBe( + "/evaluators", + ); + }); +}); diff --git a/web/src/docs/primitives.tsx b/web/src/docs/primitives.tsx new file mode 100644 index 0000000..8cbeb9c --- /dev/null +++ b/web/src/docs/primitives.tsx @@ -0,0 +1,88 @@ +// The vocabulary every docs content file is written in. Content files import only +// these, so prose never carries a class name or a styling decision — the same split +// that keeps PageHeader the sole owner of the page

. + +import type { ReactNode } from "react"; +import { Link } from "react-router-dom"; +import { Button } from "../components/ui"; +import { InfoIcon } from "../components/icons"; + +// Vertical rhythm wrapper for one tab's body. Sections inside it are plain
+// elements with their own heading, so the document outline stays flat under the +// PageHeader

that DocsPage owns. +export function DocPage({ children }: { children: ReactNode }): JSX.Element { + return
{children}
; +} + +// One titled block of prose. The heading is an

because the tab title is the +// page's

, owned by PageHeader in DocsPage. +export function DocSection({ + title, + children, +}: { + title: string; + children: ReactNode; +}): JSX.Element { + return ( +
+

{title}

+ {children} +
+ ); +} + +// A copyable command. `children` is the literal command string rather than markup so +// the clipboard write and the rendered text can never diverge. +export function CodeBlock({ children }: { children: string }): JSX.Element { + const copy = () => { + void navigator.clipboard.writeText(children); + }; + + return ( +
+ {children} + +
+ ); +} + +// An aside for a caveat worth interrupting the prose for. The icon is decorative — +// the text carries the meaning. +export function DocNote({ children }: { children: ReactNode }): JSX.Element { + return ( + + ); +} + +// A link to somewhere else in the app — another docs tab or a working surface. Wraps +// react-router's Link so docs navigation stays client-side and never reloads the SPA. +export function DocLink({ to, children }: { to: string; children: ReactNode }): JSX.Element { + return ( + + {children} + + ); +} + +// A link off the app entirely — provider dashboards and upstream docs. A plain anchor +// rather than a router Link, which would treat the URL as an in-app path. Opens in a new +// tab: this is a local workspace with unsaved editor state, and navigating away in the +// same tab would discard it. +export function ExternalLink({ + href, + children, +}: { + href: string; + children: ReactNode; +}): JSX.Element { + return ( + + {children} + + ); +} diff --git a/web/src/docs/registry.test.tsx b/web/src/docs/registry.test.tsx new file mode 100644 index 0000000..f6efdb4 --- /dev/null +++ b/web/src/docs/registry.test.tsx @@ -0,0 +1,75 @@ +// The registry is the single source for tab order, slugs, titles, and bodies, so the +// checks here are the ones that would otherwise fail at runtime as a blank tab: a +// duplicate slug (two tabs, one reachable), an empty title (an unlabelled tab), or a +// content file that throws (a blank pane under a working tab strip). +// +// A .tsx suite rather than .ts: every body may render a DocLink, which needs router +// context, so the smoke render wraps each body in a MemoryRouter. +import { afterEach, describe, expect, it } from "vitest"; +import { cleanup, render } from "@testing-library/react"; +import { MemoryRouter } from "react-router-dom"; +import { DOCS, resolveDoc } from "./registry"; + +afterEach(() => { + cleanup(); +}); + +describe("docs registry", () => { + it("lists the five documented tabs in order", () => { + // Keys leads: nothing that calls a model runs without the gateway key, so it is + // both the first thing a new user needs and the tab /docs lands on. + expect(DOCS.map((entry) => entry.slug)).toEqual([ + "keys", + "evals", + "datasets", + "runs", + "cli", + ]); + }); + + it("gives every entry a unique slug", () => { + const slugs = DOCS.map((entry) => entry.slug); + expect(new Set(slugs).size).toBe(slugs.length); + }); + + it("gives every entry a non-empty slug and title", () => { + for (const entry of DOCS) { + expect(entry.slug.length).toBeGreaterThan(0); + expect(entry.title.length).toBeGreaterThan(0); + } + }); + + it("resolves a known slug to its entry", () => { + expect(resolveDoc("runs").title).toBe("Runs"); + }); + + it("lands on Keys for the bare route", () => { + expect(resolveDoc(undefined).title).toBe("Keys"); + }); + + it("falls back to the first tab for an unknown slug", () => { + // A renamed or mistyped slug must land on readable content rather than a blank + // pane under a working tab strip. + expect(resolveDoc("does-not-exist")).toBe(DOCS[0]); + }); + + it("falls back to the first tab when no slug is given", () => { + // This is the bare /docs route, which carries no :slug param. + expect(resolveDoc(undefined)).toBe(DOCS[0]); + }); + + it.each(DOCS.map((entry) => [entry.slug, entry] as const))( + "%s renders its body without throwing", + (_slug, entry) => { + const { container } = render( + + + , + ); + + // Rendered *something*: an empty body is a blank tab, which the tab strip + // would happily present as working. + expect(container.textContent?.trim().length).toBeGreaterThan(0); + }, + ); +}); diff --git a/web/src/docs/registry.ts b/web/src/docs/registry.ts new file mode 100644 index 0000000..9ae8147 --- /dev/null +++ b/web/src/docs/registry.ts @@ -0,0 +1,40 @@ +// The single source of truth for the docs section: array order is tab order, and the +// tab strip, the routes, and the page titles all read from here. Keeping them in one +// list is what stops a tab existing with no reachable content, or a route existing with +// no tab pointing at it. +// +// Adding a page: write the content component, import it, append an entry. + +import type { ComponentType } from "react"; +import { Cli } from "./content/Cli"; +import { Datasets } from "./content/Datasets"; +import { Evals } from "./content/Evals"; +import { Keys } from "./content/Keys"; +import { Runs } from "./content/Runs"; + +export type DocEntry = { + /** URL segment under /docs. Stable — it is what people paste to each other. */ + slug: string; + /** Tab label, and the page

that DocsPage renders. */ + title: string; + /** The prose. Takes no props: docs render from static content, never from fetches. */ + Body: ComponentType; +}; + +export const DOCS: DocEntry[] = [ + // Keys leads, and so is what /docs lands on: nothing that calls a model runs until + // the gateway key exists, which makes it the first thing a new user needs. + { slug: "keys", title: "Keys", Body: Keys }, + { slug: "evals", title: "Evals", Body: Evals }, + { slug: "datasets", title: "Datasets", Body: Datasets }, + { slug: "runs", title: "Runs", Body: Runs }, + { slug: "cli", title: "CLI", Body: Cli }, +]; + +// Resolving in code rather than redirecting: an unknown slug renders the first tab and +// leaves the URL alone, so a stale link degrades to something readable instead of a +// blank pane, and no history entry is spent on the correction. The bare /docs route +// arrives here as `undefined` and lands in the same place. +export function resolveDoc(slug: string | undefined): DocEntry { + return DOCS.find((entry) => entry.slug === slug) ?? DOCS[0]; +} diff --git a/web/src/pages/DocsPage.test.tsx b/web/src/pages/DocsPage.test.tsx new file mode 100644 index 0000000..808a042 --- /dev/null +++ b/web/src/pages/DocsPage.test.tsx @@ -0,0 +1,114 @@ +// DocsPage owns routing and chrome, not content, so these tests are about which entry +// is selected and how it is reachable — never about the prose, which registry.test.tsx +// covers. +import { afterEach, describe, expect, it } from "vitest"; +import { cleanup, render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { MemoryRouter, Route, Routes } from "react-router-dom"; +import DocsPage from "./DocsPage"; +import { DOCS } from "../docs/registry"; + +afterEach(() => { + cleanup(); +}); + +// Both routes point at the same component, so the harness mounts both: the bare /docs +// path carries no :slug, which is its own resolution case. +function renderDocs(path: string) { + return render( + + + } /> + } /> + + , + ); +} + +// Content prose links to other tabs and to working surfaces, so several body links share +// a name with a tab. Every tab assertion is scoped to the strip's