Add Logfire tracing, experiment engine, and setup walkthrough

README.md

@@ -139,19 +139,39 @@ Override the default model, highest precedence first: an explicit argument,
 > OpenAI-compatible endpoint, local models — is a change to that resolution layer and the
 > config schema rather than a change to how evaluators, datasets, or runs work.
 
+## Setup
+
+`valcore serve` shows a setup card on the Overview page listing each key valcore knows
+about, whether it is currently set, and what it unlocks. Keys are never entered through
+the web UI — no secret crosses HTTP — and are always set from the CLI:
+
+```bash
+valcore config set-key                  # required: runs and generation
+valcore config set-logfire-token        # optional: sends traces to Logfire
+valcore config set-logfire-key          # optional: pushes datasets to Logfire
+```
+
+Without the gateway key, generation and runs are unavailable, and the UI shows why. Manual
+authoring, dataset upload, editing, hand-labeling, and every export still work with no key
+configured at all.
+
 ## Commands
 
 | Command | What it does |
 | --- | --- |
 | `valcore serve` | Serve the web UI and API (`--port`, `--host`, `--no-browser`). |
 | `valcore list <evaluators\|datasets\|runs>` | List resources as a table or, with `--json`, as JSON. |
 | `valcore run <evaluator> <dataset>` | Run an evaluator version over a dataset. |
+| `valcore experiment <evaluator> <dataset>` | Run an evaluator version over a dataset via `pydantic_evals.Dataset.evaluate`. |
 | `valcore export <evaluator>` | Export an evaluator (and, with `--dataset`, a dataset) as a Python script or, with `--format json`, a portable eval package. |
 | `valcore import <file>` | Import a JSON eval package back into the local database. |
 | `valcore config set-key [KEY]` | Store the gateway API key in the config file. |
+| `valcore config set-logfire-token [TOKEN]` | Store the Logfire write token in the config file. |
+| `valcore config set-logfire-key [KEY]` | Store the Logfire API key in the config file. |
 | `valcore config get` | Show the current config (the key is masked unless `--show-key`). |
 | `valcore config path` | Print the path to the config file. |
 | `valcore config edit` | Open the config file in `$EDITOR`. |
+| `valcore logfire push <dataset>` | Push a dataset to Logfire's hosted dataset store. |
 | `valcore skills install` | Install the bundled agent skills (`--claude`, `--copilot`, …). |
 | `valcore skills list` | Show the bundled skills and where each is installed. |
 | `valcore skills uninstall` | Remove the bundled skills from the selected directories. |
@@ -281,6 +301,32 @@ Exit codes:
 `--min-accuracy` requires a categorical accuracy metric; numeric or unlabeled runs have
 no accuracy and error rather than silently passing.
 
+## Logfire
+
+`logfire` is an optional extra — install it to get traces:
+
+```bash
+uv tool install 'valcore[logfire]'
+```
+
+With a Logfire token configured (see [Setup](#setup)), each `valcore run` opens a
+`valcore.run` span carrying the run's evaluator version, dataset, and concurrency, with a
+`valcore.score_row` child span per row; on close, the run span records its status and each
+agreement metric as attributes, so a Logfire query can filter runs by accuracy directly.
+The [Pydantic AI Gateway](https://ai.pydantic.dev/gateway/) already reports the LLM calls
+themselves — valcore adds only the surrounding run and row context around them, and
+deliberately does not re-report the calls, which would double-count tokens and cost.
+
+`valcore experiment <evaluator> <dataset>` runs the same evaluation through
+`pydantic_evals.Dataset.evaluate` instead of `run`'s own engine, so it also appears in
+Logfire's experiments view. It persists a run the same way `run` does, so it shows up on
+the Runs page too. Unlike `run`, it cannot be cancelled, because `Dataset.evaluate` has no
+cancellation.
+
+`valcore logfire push <dataset>` publishes a dataset to Logfire's hosted dataset store.
+It needs a Logfire API key (see [Setup](#setup)) with the `project:read_datasets` and
+`project:write_datasets` scopes.
+
 ## `~/.valcore`
 
 All state lives under `~/.valcore` (mode `0700`). Set `VALCORE_HOME` to relocate it.

pyproject.toml

@@ -32,6 +32,11 @@ dependencies = [
 [project.scripts]
 valcore = "valcore.cli:main"
 
+[project.optional-dependencies]
+# The `fastapi` sub-extra brings in opentelemetry-instrumentation-fastapi, required by
+# `logfire.instrument_fastapi` in valcore.api.main.create_app.
+logfire = ["logfire[fastapi]>=4.39,<5"]
+
 [dependency-groups]
 dev = [
     "pytest",
@@ -43,6 +48,10 @@ dev = [
     # unpinned bump would change lint results in CI and locally at once.
     "ruff>=0.16,<0.17",
     "pre-commit>=4,<5",
+    # Not a runtime dependency — the `logfire` extra. Declared here so the span tests in
+    # tests/test_tracing.py and tests/test_runner.py run rather than skipping themselves, and
+    # so create_app's logfire.instrument_fastapi call has opentelemetry-instrumentation-fastapi.
+    "logfire[fastapi]>=4.39,<5",
 ]
 
 [build-system]

src/valcore/api/main.py

@@ -4,12 +4,14 @@
 from importlib.resources import files as _package_files
 from pathlib import Path
 
+import logfire_api as logfire
 from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 from fastapi.staticfiles import StaticFiles
 
 from valcore.api.dtos import ErrorBody, ErrorResponse
+from valcore.config import load_config
 from valcore.errors import (
     ConfigError,
     ContractError,
@@ -22,6 +24,7 @@
 from valcore.models import VALID_CAPABILITIES
 from valcore.settings import MODEL_CATALOG
 from valcore.tools import tool_names
+from valcore.tracing import configure_tracing
 
 _STATUS_BY_ERROR: tuple[tuple[type[ValcoreError], int], ...] = (
     (NotFoundError, 404),
@@ -78,7 +81,7 @@ async def handler(_request: Request, exc: ValcoreError) -> JSONResponse:
 
 def _include_routers(app: FastAPI) -> None:
     """Discover and mount resource routers, tolerating ones that do not exist yet."""
-    for module_name in ("evaluators", "datasets", "runs", "overview"):
+    for module_name in ("evaluators", "datasets", "runs", "overview", "setup"):
         try:
             module = importlib.import_module(f"valcore.api.routes.{module_name}")
         except ImportError:
@@ -100,6 +103,9 @@ def create_app() -> FastAPI:
 
     _register_exception_handlers(app)
 
+    configure_tracing(load_config())
+    logfire.instrument_fastapi(app)
+
     @app.get("/api/health")
     async def health() -> dict[str, str]:
         """Liveness probe."""

src/valcore/api/routes/datasets.py

@@ -5,16 +5,18 @@
 import json
 import re
 from datetime import datetime
-from typing import Annotated
+from typing import Annotated, Literal
 
 from fastapi import APIRouter, Depends, File, Form, UploadFile
 from pydantic import BaseModel, ConfigDict
 
+from valcore import config
 from valcore.api.deps import get_store
 from valcore.config_io import EvalPackage
 from valcore.datagen import generate_rows
 from valcore.errors import ContractError
 from valcore.export import render_dataset_module, render_judge_module
+from valcore.logfire_io import push_dataset
 from valcore.models import LabelSchema, LabelSource
 from valcore.schema_migration import label_matches_schema
 from valcore.seeding import dataset_shape_from_version
@@ -99,6 +101,14 @@ class RowsAppend(BaseModel):
     rows: list[dict]
 
 
+class LogfirePushRequest(BaseModel):
+    """Request body to push a dataset to Logfire's hosted dataset store."""
+
+    name: str | None = None
+    description: str | None = None
+    on_conflict: Literal["update", "error"] = "update"
+
+
 class RowPatch(BaseModel):
     """Request body to relabel or annotate a single row."""
 
@@ -420,6 +430,7 @@ def _check_column_notes(column_notes: dict[str, str] | None, columns: list[str])
 @router.post("/generate")
 async def generate_dataset(body: DatasetGenerate, store: StoreDep) -> DatasetCreatedOut:
     """Generate a dataset and its rows with suggested labels."""
+    config.require_gateway_key()
     if body.count > _MAX_GENERATE_COUNT:
         raise ContractError(f"count may not exceed {_MAX_GENERATE_COUNT}.")
 
@@ -467,6 +478,7 @@ async def generate_dataset_from_version(
     body: DatasetGenerateFromVersion, store: StoreDep
 ) -> DatasetCreatedOut:
     """Generate a dataset shaped by an evaluator version, runnable against it by construction."""
+    config.require_gateway_key()
     if body.count > _MAX_GENERATE_COUNT:
         raise ContractError(f"count may not exceed {_MAX_GENERATE_COUNT}.")
 
@@ -545,6 +557,7 @@ async def generate_more_rows(id: str, body: RowsGenerate, store: StoreDep) -> li
     the new rows stay compatible with the existing ones and with any evaluator that already
     runs against them.
     """
+    config.require_gateway_key()
     if body.count > _MAX_GENERATE_COUNT:
         raise ContractError(f"count may not exceed {_MAX_GENERATE_COUNT}.")
 
@@ -731,3 +744,17 @@ async def export_dataset_json(
         agent_filename = f"{stem}.agent.json" if split else f"{stem}.json"
         files["valcore_judge.py"] = render_judge_module(version, agent_filename)
     return ExportFilesResponse(files=files)
+
+
+@router.post("/{id}/logfire/push")
+async def push_dataset_to_logfire(id: str, body: LogfirePushRequest, store: StoreDep) -> dict:
+    """Push a dataset and its rows to Logfire's hosted dataset store."""
+    dataset = store.get_dataset(id)
+    rows = store.list_rows(id)
+    return await push_dataset(
+        dataset,
+        rows,
+        name=body.name,
+        description=body.description,
+        on_conflict=body.on_conflict,
+    )

src/valcore/api/routes/evaluators.py

@@ -7,7 +7,7 @@
 from fastapi import APIRouter, Depends
 from pydantic import BaseModel, ConfigDict
 
-from valcore import generator
+from valcore import config, generator
 from valcore.api.deps import get_store
 from valcore.config_io import EvalPackage
 from valcore.errors import ContractError, FrozenVersionError
@@ -405,6 +405,7 @@ async def generate(body: GenerateRequest, store: StoreDep) -> GeneratedConfig:
     Calls an LLM and can take tens of seconds; the UI presents the returned config as an
     editable draft saved as a version separately.
     """
+    config.require_gateway_key()
     columns, label_schema = _resolve_seed(body, store)
     return await generator.generate_config(
         body.criteria,
@@ -422,6 +423,7 @@ async def generate_version(id: str, body: GenerateRequest, store: StoreDep) -> G
     of seconds; the UI presents the returned config as an editable draft that the user
     saves as a version separately.
     """
+    config.require_gateway_key()
     store.get_evaluator(id)
     columns, label_schema = _resolve_seed(body, store)
     return await generator.generate_config(
@@ -439,4 +441,5 @@ async def refine_version(body: RefineRequest) -> RefinedConfig:
     Calls an LLM and can take tens of seconds. The response includes ``changed_fields``
     for the diff view; nothing is saved until the user submits a new version.
     """
+    config.require_gateway_key()
     return await generator.refine_config(body.config, body.instruction)

src/valcore/api/routes/runs.py

@@ -16,6 +16,7 @@
 from pydantic_ai import Agent
 from sse_starlette.sse import EventSourceResponse
 
+from valcore import config
 from valcore.api.deps import get_store
 from valcore.api.events import bus
 from valcore.errors import ContractError
@@ -149,12 +150,17 @@ async def _run_to_completion(
 
     Any exception escaping the runner is recorded on the run as ``FAILED`` so a run
     is never left stuck in ``RUNNING``. The bus is always closed so subscribers stop.
+
+    Guards on ``config.require_gateway_key()`` before ever building an agent or calling
+    ``execute_run``: ``defer_model_check=True`` lets ``build_agent`` succeed with no key,
+    so without this a keyless run would instead fail once per row inside ``_score_row``.
     """
 
     async def on_event(event: RunEvent) -> None:
         bus.publish(run_id, {"type": event.type, "run_id": run_id, "payload": event.payload})
 
     try:
+        config.require_gateway_key()
         agent: Agent | None = None
         if agent_factory is not None:
             run = await asyncio.to_thread(store.get_run, run_id)
@@ -231,7 +237,12 @@ async def create_run(body: RunCreate, store: StoreDep, agent_factory: AgentFacto
     """Create a run (freezing its version) and launch it in the background.
 
     Returns immediately with status ``PENDING``; the run is never awaited in the handler.
+
+    Guards on ``config.require_gateway_key()`` before ``store.create_run``: a missing key
+    must surface synchronously as a ``ConfigError``, with no run ever persisted, rather than
+    as an asynchronous ``FAILED`` transition discovered by polling.
     """
+    config.require_gateway_key()
     run = store.create_run(
         kind=body.kind,
         version_id=body.version_id,
@@ -397,7 +408,12 @@ async def retry_failed(id: str, store: StoreDep, agent_factory: AgentFactoryDep)
 
     The run is reset to ``PENDING`` before relaunching so a client polling status is
     never fooled by the previous run's stale terminal state.
+
+    Guards on ``config.require_gateway_key()`` before any other work: a missing key
+    must surface synchronously as a ``ConfigError``, with the prior run's status and
+    results untouched and no background task launched.
     """
+    config.require_gateway_key()
     store.get_run(id)
     failed_row_ids = store.failed_result_row_ids(id)
     run = store.update_run_status(id, RunStatus.PENDING, error=None, finished_at=None)

src/valcore/api/routes/setup.py

@@ -0,0 +1,70 @@
+"""Read-only setup status: which configuration keys are effectively set.
+
+Keys are written only via the CLI (``valcore config set-key`` and its Logfire siblings), so
+there is no POST here and no key value ever crosses this endpoint -- only booleans, computed
+from the same ``*_present`` helpers ``require_gateway_key`` relies on, so presence reflects an
+exported env var exactly as it does everywhere else in the codebase.
+"""
+
+from fastapi import APIRouter
+from pydantic import BaseModel
+
+from valcore.config import (
+    gateway_key_present,
+    load_config,
+    logfire_api_key_present,
+    logfire_token_present,
+)
+
+router = APIRouter(prefix="/api/setup", tags=["setup"])
+
+
+class KeyStatus(BaseModel):
+    """Presence and static metadata for one configuration key. Never carries its value."""
+
+    name: str
+    set: bool
+    required: bool
+    label: str
+    command: str
+    purpose: str
+
+
+class SetupOut(BaseModel):
+    """The full setup status: one entry per documented configuration key."""
+
+    keys: list[KeyStatus]
+
+
+@router.get("", response_model=SetupOut)
+async def get_setup() -> SetupOut:
+    """Report effective presence for the gateway key and both Logfire credentials."""
+    cfg = load_config()
+    return SetupOut(
+        keys=[
+            KeyStatus(
+                name="gateway_api_key",
+                set=gateway_key_present(cfg),
+                required=True,
+                label="Pydantic AI Gateway key",
+                command="valcore config set-key",
+                purpose="Runs evaluators and generates evaluators and datasets.",
+            ),
+            KeyStatus(
+                name="logfire_token",
+                set=logfire_token_present(cfg),
+                required=False,
+                label="Logfire write token",
+                command="valcore config set-logfire-token",
+                purpose="Sends run traces to Logfire.",
+            ),
+            KeyStatus(
+                name="logfire_api_key",
+                set=logfire_api_key_present(cfg),
+                required=False,
+                label="Logfire API key",
+                command="valcore config set-logfire-key",
+                purpose="Pushes datasets to Logfire's hosted store.",
+            ),
+        ]
+    )