Proxy and sync every LLM provider through one definition

.gitignore

@@ -28,6 +28,7 @@ coverage.xml
 # IDE / editor
 .idea/
 .vscode/
+.zed/
 *.swp
 *.swo
 

runner/src/coval_bench/api/app.py

@@ -49,7 +49,7 @@
     arena,
     health,
     leaderboard,
-    llm_phonely,
+    llm_proxy,
     mocktools,
     pricing,
     providers,
@@ -63,6 +63,7 @@
 from coval_bench.db.conn import lifespan_pool
 from coval_bench.fixture_sources import install_fixture_providers
 from coval_bench.llm.phonely import PhonelyClient
+from coval_bench.llm.turn import TurnClient
 from coval_bench.logging import configure_logging
 from coval_bench.mocktools.dispatch import build_dispatcher
 
@@ -120,17 +121,17 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
                 logger.warning("posthog_init_failed", exc_info=True)
                 posthog_client = None
         app.state.posthog = posthog_client
-        phonely_client: PhonelyClient | None = None
+        llm_clients: dict[str, TurnClient] = {}
         phonely_key = resolved.phonely_api_key
         if phonely_key and phonely_key.get_secret_value() and resolved.phonely_agent_id:
-            phonely_client = PhonelyClient(
+            llm_clients["phonely"] = PhonelyClient(
                 phonely_key.get_secret_value(),
                 resolved.phonely_agent_id,
                 resolved.phonely_base_url,
             )
         else:
-            logger.info("phonely_proxy_disabled")
-        app.state.phonely_client = phonely_client
+            logger.info("llm_proxy_disabled", provider="phonely")
+        app.state.llm_clients = llm_clients
         # Built here rather than on first request: loading and cross-checking the
         # fixtures inside a live call would put that cost on the agent's turn.
         # Absent fixtures are normal in CI and a fresh checkout, so the route
@@ -150,8 +151,8 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
                 app.state.settings = resolved
                 yield
         finally:
-            if phonely_client is not None:
-                await phonely_client.aclose()
+            for llm_client in llm_clients.values():
+                await llm_client.aclose()
             if posthog_client is not None:
                 try:
                     posthog_client.shutdown()  # type: ignore[no-untyped-call]
@@ -221,7 +222,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     # public read API. slowapi carries no default limit, so /mock and /llm are exempt
     # by construction — a 429 mid-conversation would be graded as the agent failing.
     app.include_router(mocktools.router)
-    app.include_router(llm_phonely.router)
+    app.include_router(llm_proxy.router)
 
     # Serve locally-generated arena clips when no external audio host is set
     # (prod sets arena_gcs_bucket for GCS, or arena_audio_base_url for a CDN origin).

runner/src/coval_bench/api/deps.py

@@ -26,7 +26,8 @@
 from coval_bench.api import clerk
 from coval_bench.config import Settings
 from coval_bench.db.registry_store import fetch_models
-from coval_bench.llm.phonely import PhonelyClient
+from coval_bench.llm.benchmark import LLM_MODELS
+from coval_bench.llm.turn import TurnClient
 from coval_bench.registries import RegisteredModel
 
 logger = structlog.get_logger("coval_bench.api")
@@ -68,11 +69,14 @@ def secret_matches(provided: str | None, expected: SecretStr | None) -> bool:
     return bool(value) and hmac.compare_digest(provided.encode(), value.encode())
 
 
-def get_phonely_client(request: Request) -> PhonelyClient:
-    """Return the lifespan-owned Phonely client, or fail closed."""
-    client: PhonelyClient | None = getattr(request.app.state, "phonely_client", None)
+def get_turn_client(provider: str, request: Request) -> TurnClient:
+    """Return the lifespan-owned client for this provider, or fail closed."""
+    if provider not in LLM_MODELS:
+        raise HTTPException(404, f"{provider} is not an LLM benchmark provider")
+    clients: dict[str, TurnClient] = getattr(request.app.state, "llm_clients", {})
+    client = clients.get(provider)
     if client is None:
-        raise HTTPException(503, "Phonely proxy is not configured")
+        raise HTTPException(503, f"{provider} proxy is not configured")
     return client
 
 

runner/src/coval_bench/api/routers/llm_phonely.py → runner/src/coval_bench/api/routers/llm_proxy.py

@@ -1,7 +1,7 @@
 # Copyright 2026 The Coval Benchmarks Authors
 # SPDX-License-Identifier: Apache-2.0
 
-"""Authenticated OpenAI-compatible proxy for Phonely text-agent turns."""
+"""Authenticated OpenAI-compatible proxy for LLM benchmark turns."""
 
 from __future__ import annotations
 
@@ -11,32 +11,33 @@
 from typing import Any
 
 import structlog
-from fastapi import APIRouter, BackgroundTasks, Depends, Header, HTTPException
+from fastapi import APIRouter, BackgroundTasks, Depends, Header, HTTPException, Path
 from psycopg_pool import AsyncConnectionPool
 from pydantic import BaseModel, Field
 from starlette.responses import JSONResponse
 
 from coval_bench.api.deps import (
     bearer_token,
-    get_phonely_client,
     get_pool,
     get_settings,
+    get_turn_client,
     secret_matches,
 )
 from coval_bench.config import Settings
 from coval_bench.db.llm_turns import insert_turn
-from coval_bench.llm.phonely import MODEL, PROVIDER, PhonelyClient, PhonelyError, TurnResult
+from coval_bench.llm.benchmark import LLM_MODELS
+from coval_bench.llm.turn import TurnClient, TurnError, TurnResult
 
-logger = structlog.get_logger("coval_bench.api.llm_phonely")
+logger = structlog.get_logger("coval_bench.api.llm_proxy")
 
-router = APIRouter(prefix="/llm/phonely", tags=["llm-phonely"])
+router = APIRouter(prefix="/llm/{provider}", tags=["llm-proxy"])
 
 _MESSAGE_KEYS = frozenset({"role", "content", "name", "tool_calls", "tool_call_id"})
 _TURN_TIMEOUT_S = 150.0
 
 
-# Only model (the Phonely callId) and messages reach Phonely; its agent owns tools and
-# sampling, so other OpenAI request fields are dropped.
+# Only model (the provider's session id) and messages reach the provider; its agent
+# owns tools and sampling, so other OpenAI request fields are dropped.
 class ChatRequest(BaseModel):
     model: str = Field(min_length=1)
     messages: list[dict[str, Any]]
@@ -69,6 +70,7 @@ def _messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
 async def _record_turn(
     pool: AsyncConnectionPool[Any],
     *,
+    provider: str,
     simulation_id: str,
     turn_index: int,
     result: TurnResult,
@@ -78,8 +80,8 @@ async def _record_turn(
             pool,
             simulation_id=simulation_id,
             turn_index=turn_index,
-            provider=PROVIDER,
-            model=MODEL,
+            provider=provider,
+            model=LLM_MODELS[provider],
             ttft_ms=result.ttft_ms,
             total_ms=result.total_ms,
             output_tokens=result.output_tokens,
@@ -122,21 +124,23 @@ def _completion(call_id: str, result: TurnResult) -> dict[str, Any]:
 
 @router.post("/session", dependencies=[Depends(require_proxy_secret)])
 async def create_session(
-    client: PhonelyClient = Depends(get_phonely_client),
+    provider: str = Path(),
+    client: TurnClient = Depends(get_turn_client),
 ) -> dict[str, str | None]:
     try:
         session = await client.create_session()
-    except PhonelyError as exc:
-        logger.warning("phonely_session_failed", error=str(exc))
-        raise HTTPException(502, "Phonely session creation failed") from exc
+    except TurnError as exc:
+        logger.warning("llm_session_failed", provider=provider, error=str(exc))
+        raise HTTPException(502, f"{provider} session creation failed") from exc
     return {"sessionId": session.call_id, "expiresAt": session.expires_at}
 
 
 @router.post("/chat", dependencies=[Depends(require_proxy_secret)])
 async def chat(
     body: ChatRequest,
     background: BackgroundTasks,
-    client: PhonelyClient = Depends(get_phonely_client),
+    provider: str = Path(),
+    client: TurnClient = Depends(get_turn_client),
     pool: AsyncConnectionPool[Any] = Depends(get_pool),
 ) -> JSONResponse:
     if body.stream:
@@ -147,22 +151,23 @@ async def chat(
         async with asyncio.timeout(_TURN_TIMEOUT_S):
             result = await client.stream_turn(body.model, messages)
     except TimeoutError as exc:
-        logger.warning("phonely_turn_timed_out", turn_index=turn_index)
-        raise HTTPException(504, "Phonely completion timed out") from exc
-    except PhonelyError as exc:
-        logger.warning("phonely_turn_failed", turn_index=turn_index, error=str(exc))
-        raise HTTPException(502, "Phonely completion failed") from exc
+        logger.warning("llm_turn_timed_out", provider=provider, turn_index=turn_index)
+        raise HTTPException(504, f"{provider} completion timed out") from exc
+    except TurnError as exc:
+        logger.warning("llm_turn_failed", provider=provider, turn_index=turn_index, error=str(exc))
+        raise HTTPException(502, f"{provider} completion failed") from exc
 
     if body.simulation_id:
         background.add_task(
             _record_turn,
             pool,
+            provider=provider,
             simulation_id=body.simulation_id,
             turn_index=turn_index,
             result=result,
         )
     else:
-        logger.warning("llm_turn_unattributed", turn_index=turn_index)
+        logger.warning("llm_turn_unattributed", provider=provider, turn_index=turn_index)
     return JSONResponse(
         _completion(body.model, result),
         headers={

runner/src/coval_bench/llm/benchmark.py

@@ -8,6 +8,7 @@
 from collections.abc import Iterable
 from typing import Any
 
+LLM_MODELS = {"phonely": "phonely-agent"}
 DEFAULT_PERSONA_ID = "PN3xgmsqeLDjsNNEA2e55e"
 ITERATION_COUNT = 1
 TEMPLATE_MANAGED = ("agent_ids", "persona_ids", "test_set_ids", "metric_ids", "iteration_count")