[BENCH-839] Drive the LLM roster from the models table

runner/src/coval_bench/api/app.py

@@ -61,9 +61,9 @@
 )
 from coval_bench.config import Settings, get_settings
 from coval_bench.db.conn import lifespan_pool
+from coval_bench.db.registry_store import fetch_models
 from coval_bench.fixture_sources import install_fixture_providers
-from coval_bench.llm.phonely import PhonelyClient
-from coval_bench.llm.turn import TurnClient
+from coval_bench.llm.benchmark import llm_models, make_clients
 from coval_bench.logging import configure_logging
 from coval_bench.mocktools.dispatch import build_dispatcher
 
@@ -121,16 +121,8 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
                 logger.warning("posthog_init_failed", exc_info=True)
                 posthog_client = None
         app.state.posthog = posthog_client
-        llm_clients: dict[str, TurnClient] = {}
-        phonely_key = resolved.phonely_api_key
-        if phonely_key and phonely_key.get_secret_value() and resolved.phonely_agent_id:
-            llm_clients["phonely"] = PhonelyClient(
-                phonely_key.get_secret_value(),
-                resolved.phonely_agent_id,
-                resolved.phonely_base_url,
-            )
-        else:
-            logger.info("llm_proxy_disabled", provider="phonely")
+        llm_clients = make_clients(resolved)
+        logger.info("llm_clients_ready", providers=sorted(llm_clients))
         app.state.llm_clients = llm_clients
         # Built here rather than on first request: loading and cross-checking the
         # fixtures inside a live call would put that cost on the agent's turn.
@@ -149,6 +141,14 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
             async with lifespan_pool(resolved) as pool:
                 app.state.pool = pool
                 app.state.settings = resolved
+                try:
+                    roster = await fetch_models(pool)
+                except Exception:
+                    logger.warning("llm_client_check_skipped", exc_info=True)
+                else:
+                    for model in llm_models(roster):
+                        if model.collected and model.provider not in llm_clients:
+                            logger.error("llm_client_missing", provider=model.provider)
                 yield
         finally:
             for llm_client in llm_clients.values():

runner/src/coval_bench/api/deps.py

@@ -26,7 +26,7 @@
 from coval_bench.api import clerk
 from coval_bench.config import Settings
 from coval_bench.db.registry_store import fetch_models
-from coval_bench.llm.benchmark import LLM_MODELS
+from coval_bench.llm.benchmark import ProxiedModel, llm_models
 from coval_bench.llm.turn import TurnClient
 from coval_bench.registries import RegisteredModel
 
@@ -69,17 +69,6 @@ def secret_matches(provided: str | None, expected: SecretStr | None) -> bool:
     return bool(value) and hmac.compare_digest(provided.encode(), value.encode())
 
 
-def get_turn_client(provider: str, request: Request) -> TurnClient:
-    """Return the lifespan-owned client for this provider, or fail closed."""
-    if provider not in LLM_MODELS:
-        raise HTTPException(404, f"{provider} is not an LLM benchmark provider")
-    clients: dict[str, TurnClient] = getattr(request.app.state, "llm_clients", {})
-    client = clients.get(provider)
-    if client is None:
-        raise HTTPException(503, f"{provider} proxy is not configured")
-    return client
-
-
 def require_coval_admin(
     authorization: str | None = Header(default=None),
     settings: Settings = Depends(get_settings),
@@ -143,6 +132,24 @@ async def get_models(
         raise HTTPException(503, "the model registry is unavailable") from exc
 
 
+async def get_proxied_model(
+    provider: str,
+    request: Request,
+    models: list[RegisteredModel] = Depends(get_models),
+) -> ProxiedModel:
+    """The collected LLM model behind a /llm/{provider} route, or fail closed."""
+    registered = next(
+        (m for m in llm_models(models) if m.provider == provider and m.collected), None
+    )
+    if registered is None:
+        raise HTTPException(404, f"{provider} is not a collected LLM benchmark model")
+    clients: dict[str, TurnClient] = getattr(request.app.state, "llm_clients", {})
+    client = clients.get(provider)
+    if client is None:
+        raise HTTPException(503, f"{provider} proxy is not configured")
+    return ProxiedModel(provider=provider, model=registered.model, client=client)
+
+
 def get_cache_locks(request: Request) -> defaultdict[Any, asyncio.Lock]:
     """Return the per-app cache-key locks from app state."""
     return cast("defaultdict[Any, asyncio.Lock]", request.app.state.cache_locks)

runner/src/coval_bench/api/routers/llm_proxy.py

@@ -11,22 +11,22 @@
 from typing import Any
 
 import structlog
-from fastapi import APIRouter, BackgroundTasks, Depends, Header, HTTPException, Path
+from fastapi import APIRouter, BackgroundTasks, Depends, Header, HTTPException
 from psycopg_pool import AsyncConnectionPool
 from pydantic import BaseModel, Field
 from starlette.responses import JSONResponse
 
 from coval_bench.api.deps import (
     bearer_token,
     get_pool,
+    get_proxied_model,
     get_settings,
-    get_turn_client,
     secret_matches,
 )
 from coval_bench.config import Settings
 from coval_bench.db.llm_turns import insert_turn
-from coval_bench.llm.benchmark import LLM_MODELS
-from coval_bench.llm.turn import TurnClient, TurnError, TurnResult
+from coval_bench.llm.benchmark import ProxiedModel
+from coval_bench.llm.turn import TurnError, TurnResult
 
 logger = structlog.get_logger("coval_bench.api.llm_proxy")
 
@@ -70,7 +70,7 @@ def _messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
 async def _record_turn(
     pool: AsyncConnectionPool[Any],
     *,
-    provider: str,
+    target: ProxiedModel,
     simulation_id: str,
     turn_index: int,
     result: TurnResult,
@@ -80,8 +80,8 @@ async def _record_turn(
             pool,
             simulation_id=simulation_id,
             turn_index=turn_index,
-            provider=provider,
-            model=LLM_MODELS[provider],
+            provider=target.provider,
+            model=target.model,
             ttft_ms=result.ttft_ms,
             total_ms=result.total_ms,
             output_tokens=result.output_tokens,
@@ -124,32 +124,31 @@ def _completion(call_id: str, result: TurnResult) -> dict[str, Any]:
 
 @router.post("/session", dependencies=[Depends(require_proxy_secret)])
 async def create_session(
-    provider: str = Path(),
-    client: TurnClient = Depends(get_turn_client),
+    target: ProxiedModel = Depends(get_proxied_model),
 ) -> dict[str, str | None]:
     try:
-        session = await client.create_session()
+        session = await target.client.create_session()
     except TurnError as exc:
-        logger.warning("llm_session_failed", provider=provider, error=str(exc))
-        raise HTTPException(502, f"{provider} session creation failed") from exc
+        logger.warning("llm_session_failed", provider=target.provider, error=str(exc))
+        raise HTTPException(502, f"{target.provider} session creation failed") from exc
     return {"sessionId": session.call_id, "expiresAt": session.expires_at}
 
 
 @router.post("/chat", dependencies=[Depends(require_proxy_secret)])
 async def chat(
     body: ChatRequest,
     background: BackgroundTasks,
-    provider: str = Path(),
-    client: TurnClient = Depends(get_turn_client),
+    target: ProxiedModel = Depends(get_proxied_model),
     pool: AsyncConnectionPool[Any] = Depends(get_pool),
 ) -> JSONResponse:
     if body.stream:
         raise HTTPException(400, "streaming responses are not supported")
+    provider = target.provider
     messages = _messages(body.messages)
     turn_index = sum(message.get("role") == "assistant" for message in messages)
     try:
         async with asyncio.timeout(_TURN_TIMEOUT_S):
-            result = await client.stream_turn(body.model, messages)
+            result = await target.client.stream_turn(body.model, messages)
     except TimeoutError as exc:
         logger.warning("llm_turn_timed_out", provider=provider, turn_index=turn_index)
         raise HTTPException(504, f"{provider} completion timed out") from exc
@@ -161,7 +160,7 @@ async def chat(
         background.add_task(
             _record_turn,
             pool,
-            provider=provider,
+            target=target,
             simulation_id=body.simulation_id,
             turn_index=turn_index,
             result=result,

runner/src/coval_bench/llm/benchmark.py

@@ -5,10 +5,19 @@
 
 from __future__ import annotations
 
-from collections.abc import Iterable
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
 from typing import Any
 
-LLM_MODELS = {"phonely": "phonely-agent"}
+from coval_bench.config import Settings
+from coval_bench.llm.phonely import PhonelyClient
+from coval_bench.llm.turn import TurnClient
+from coval_bench.registries.benchmarks import Benchmark
+from coval_bench.registries.models import RegisteredModel
+
+CLIENT_FACTORIES: dict[str, Callable[[Settings], TurnClient | None]] = {
+    "phonely": PhonelyClient.from_settings,
+}
 DEFAULT_PERSONA_ID = "PN3xgmsqeLDjsNNEA2e55e"
 ITERATION_COUNT = 1
 TEMPLATE_MANAGED = ("agent_ids", "persona_ids", "test_set_ids", "metric_ids", "iteration_count")
@@ -19,6 +28,26 @@
 }
 
 
+@dataclass(frozen=True)
+class ProxiedModel:
+    provider: str
+    model: str
+    client: TurnClient
+
+
+def llm_models(models: Iterable[RegisteredModel]) -> list[RegisteredModel]:
+    return [model for model in models if model.benchmark is Benchmark.LLM]
+
+
+def make_clients(settings: Settings) -> dict[str, TurnClient]:
+    clients: dict[str, TurnClient] = {}
+    for provider, factory in CLIENT_FACTORIES.items():
+        client = factory(settings)
+        if client is not None:
+            clients[provider] = client
+    return clients
+
+
 def run_template_body(
     display_name: str, agent_id: str, test_set_id: str, metric_id: str
 ) -> dict[str, Any]:

runner/src/coval_bench/llm/coval_agent.py

@@ -5,19 +5,24 @@
 
 from __future__ import annotations
 
+import asyncio
 import json
 from dataclasses import dataclass, field
 from typing import Any, Self
 
 import click
 import httpx
+import psycopg
 import structlog
 from pydantic import BaseModel, SecretStr
 
 from coval_bench.config import Settings, get_settings
+from coval_bench.db.conn import lifespan_pool
+from coval_bench.db.registry_store import fetch_models
 from coval_bench.llm import benchmark
 from coval_bench.logging import configure_logging
 from coval_bench.platform_assets import COVAL_API_BASE, COVAL_API_KEY, CovalClient, SyncError, plan
+from coval_bench.registries.models import RegisteredModel
 from coval_bench.variants.platforms import redact
 
 # The public API rejects MODEL_TYPE_TEXT; CHAT is the HTTP text simulator.
@@ -38,6 +43,7 @@ class CovalTextAgentDefinition(BaseModel, frozen=True):
     proxy_secret: SecretStr
     test_set_id: str
     instruction_metric_id: str
+    collected: bool = True
 
     @property
     def customer_agent_id(self) -> str:
@@ -53,7 +59,12 @@ def run_name(self) -> str:
 
     @classmethod
     def from_settings(
-        cls, provider: str, settings: Settings, *, test_set_id: str | None = None
+        cls,
+        provider: str,
+        settings: Settings,
+        *,
+        test_set_id: str | None = None,
+        collected: bool = True,
     ) -> Self:
         proxy_url = settings.llm_proxy_public_url
         proxy_secret = settings.llm_proxy_secret
@@ -77,6 +88,7 @@ def from_settings(
             proxy_secret=proxy_secret,
             test_set_id=dental,
             instruction_metric_id=metric,
+            collected=collected,
         )
 
     def agent_body(self) -> dict[str, Any]:
@@ -118,16 +130,24 @@ def run_template_body(self, agent_id: str) -> dict[str, Any]:
         )
 
 
-def scheduled_run_body(run_name: str, run_template_id: str) -> dict[str, Any]:
+def scheduled_run_body(run_name: str, run_template_id: str, *, enabled: bool) -> dict[str, Any]:
     return {
         "display_name": run_name,
         "run_template_id": run_template_id,
         "schedule_expression": SCHEDULE_EXPRESSION,
         "schedule_timezone": SCHEDULE_TIMEZONE,
-        "enabled": True,
+        "enabled": enabled,
     }
 
 
+def load_llm_models(settings: Settings) -> list[RegisteredModel]:
+    async def _load() -> list[RegisteredModel]:
+        async with lifespan_pool(settings) as pool:
+            return benchmark.llm_models(await fetch_models(pool))
+
+    return asyncio.run(_load())
+
+
 class CovalTextClient(CovalClient):
     def __enter__(self) -> Self:
         return self
@@ -169,6 +189,11 @@ def create_scheduled_run(self, body: dict[str, Any]) -> dict[str, Any]:
         scheduled = payload.get("scheduled_run")
         return scheduled if isinstance(scheduled, dict) else payload
 
+    def update_scheduled_run(self, scheduled_id: str, body: dict[str, Any]) -> dict[str, Any]:
+        payload = self._request("PATCH", f"/scheduled-runs/{scheduled_id}", body)
+        scheduled = payload.get("scheduled_run")
+        return scheduled if isinstance(scheduled, dict) else payload
+
 
 @dataclass
 class SyncResult:
@@ -234,12 +259,19 @@ def sync(
             result.actions.append("run template: unchanged")
     template_id = str(template["id"])
 
-    if client.find_scheduled_run(template_id) is None:
+    scheduled = client.find_scheduled_run(template_id)
+    if scheduled is None:
         result.actions.append("scheduled run: create")
         if not dry_run:
-            client.create_scheduled_run(scheduled_run_body(definition.run_name, template_id))
+            client.create_scheduled_run(
+                scheduled_run_body(definition.run_name, template_id, enabled=definition.collected)
+            )
+    elif bool(scheduled.get("enabled")) != definition.collected:
+        result.actions.append(f"scheduled run: {'enable' if definition.collected else 'disable'}")
+        if not dry_run:
+            client.update_scheduled_run(str(scheduled["id"]), {"enabled": definition.collected})
     else:
-        result.actions.append("scheduled run: exists")
+        result.actions.append("scheduled run: unchanged")
     return result
 
 
@@ -263,8 +295,10 @@ def sync_llm(dry_run: bool, test_set_id: str | None, coval_api_base: str) -> Non
     run_logger = structlog.get_logger("coval_bench.llm.coval_agent")
     try:
         definitions = [
-            CovalTextAgentDefinition.from_settings(provider, settings, test_set_id=test_set_id)
-            for provider in benchmark.LLM_MODELS
+            CovalTextAgentDefinition.from_settings(
+                model.provider, settings, test_set_id=test_set_id, collected=model.collected
+            )
+            for model in load_llm_models(settings)
         ]
         if dry_run:
             for definition in definitions:
@@ -274,7 +308,7 @@ def sync_llm(dry_run: bool, test_set_id: str | None, coval_api_base: str) -> Non
                 definition.provider: sync(client, definition, dry_run=dry_run)
                 for definition in definitions
             }
-    except (SyncError, RuntimeError, httpx.HTTPError) as exc:
+    except (SyncError, RuntimeError, httpx.HTTPError, psycopg.Error) as exc:
         if not dry_run:
             run_logger.error("RUN_FAILED", error=str(exc), exc_info=exc)
         raise click.ClickException(str(exc)) from exc
@@ -287,8 +321,11 @@ def sync_llm(dry_run: bool, test_set_id: str | None, coval_api_base: str) -> Non
     if dry_run:
         return
     fetchable: dict[str, str] = {}
-    for provider, result in results.items():
-        if "scheduled run: create" in result.actions:
+    for definition, result in zip(definitions, results.values(), strict=True):
+        provider = definition.provider
+        if not definition.collected:
+            run_logger.info("llm_sync_fetch_skipped", provider=provider, reason="not_collected")
+        elif "scheduled run: create" in result.actions:
             run_logger.info(
                 "llm_sync_fetch_deferred", provider=provider, reason="scheduled_run_created"
             )