diff --git a/.gitignore b/.gitignore index d234127a..143b26db 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ coverage.xml # IDE / editor .idea/ .vscode/ +.zed/ *.swp *.swo diff --git a/runner/src/coval_bench/api/app.py b/runner/src/coval_bench/api/app.py index 97f2e8bf..48211720 100644 --- a/runner/src/coval_bench/api/app.py +++ b/runner/src/coval_bench/api/app.py @@ -49,7 +49,7 @@ arena, health, leaderboard, - llm_phonely, + llm_proxy, mocktools, pricing, providers, @@ -63,6 +63,7 @@ from coval_bench.db.conn import lifespan_pool from coval_bench.fixture_sources import install_fixture_providers from coval_bench.llm.phonely import PhonelyClient +from coval_bench.llm.turn import TurnClient from coval_bench.logging import configure_logging from coval_bench.mocktools.dispatch import build_dispatcher @@ -120,17 +121,17 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: logger.warning("posthog_init_failed", exc_info=True) posthog_client = None app.state.posthog = posthog_client - phonely_client: PhonelyClient | None = None + llm_clients: dict[str, TurnClient] = {} phonely_key = resolved.phonely_api_key if phonely_key and phonely_key.get_secret_value() and resolved.phonely_agent_id: - phonely_client = PhonelyClient( + llm_clients["phonely"] = PhonelyClient( phonely_key.get_secret_value(), resolved.phonely_agent_id, resolved.phonely_base_url, ) else: - logger.info("phonely_proxy_disabled") - app.state.phonely_client = phonely_client + logger.info("llm_proxy_disabled", provider="phonely") + app.state.llm_clients = llm_clients # Built here rather than on first request: loading and cross-checking the # fixtures inside a live call would put that cost on the agent's turn. # Absent fixtures are normal in CI and a fresh checkout, so the route @@ -150,8 +151,8 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: app.state.settings = resolved yield finally: - if phonely_client is not None: - await phonely_client.aclose() + for llm_client in llm_clients.values(): + await llm_client.aclose() if posthog_client is not None: try: posthog_client.shutdown() # type: ignore[no-untyped-call] @@ -221,7 +222,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # public read API. slowapi carries no default limit, so /mock and /llm are exempt # by construction — a 429 mid-conversation would be graded as the agent failing. app.include_router(mocktools.router) - app.include_router(llm_phonely.router) + app.include_router(llm_proxy.router) # Serve locally-generated arena clips when no external audio host is set # (prod sets arena_gcs_bucket for GCS, or arena_audio_base_url for a CDN origin). diff --git a/runner/src/coval_bench/api/deps.py b/runner/src/coval_bench/api/deps.py index 0fb45b7b..cb4a96ec 100644 --- a/runner/src/coval_bench/api/deps.py +++ b/runner/src/coval_bench/api/deps.py @@ -26,7 +26,8 @@ from coval_bench.api import clerk from coval_bench.config import Settings from coval_bench.db.registry_store import fetch_models -from coval_bench.llm.phonely import PhonelyClient +from coval_bench.llm.benchmark import LLM_MODELS +from coval_bench.llm.turn import TurnClient from coval_bench.registries import RegisteredModel logger = structlog.get_logger("coval_bench.api") @@ -68,11 +69,14 @@ def secret_matches(provided: str | None, expected: SecretStr | None) -> bool: return bool(value) and hmac.compare_digest(provided.encode(), value.encode()) -def get_phonely_client(request: Request) -> PhonelyClient: - """Return the lifespan-owned Phonely client, or fail closed.""" - client: PhonelyClient | None = getattr(request.app.state, "phonely_client", None) +def get_turn_client(provider: str, request: Request) -> TurnClient: + """Return the lifespan-owned client for this provider, or fail closed.""" + if provider not in LLM_MODELS: + raise HTTPException(404, f"{provider} is not an LLM benchmark provider") + clients: dict[str, TurnClient] = getattr(request.app.state, "llm_clients", {}) + client = clients.get(provider) if client is None: - raise HTTPException(503, "Phonely proxy is not configured") + raise HTTPException(503, f"{provider} proxy is not configured") return client diff --git a/runner/src/coval_bench/api/routers/llm_phonely.py b/runner/src/coval_bench/api/routers/llm_proxy.py similarity index 75% rename from runner/src/coval_bench/api/routers/llm_phonely.py rename to runner/src/coval_bench/api/routers/llm_proxy.py index df764747..4f311fee 100644 --- a/runner/src/coval_bench/api/routers/llm_phonely.py +++ b/runner/src/coval_bench/api/routers/llm_proxy.py @@ -1,7 +1,7 @@ # Copyright 2026 The Coval Benchmarks Authors # SPDX-License-Identifier: Apache-2.0 -"""Authenticated OpenAI-compatible proxy for Phonely text-agent turns.""" +"""Authenticated OpenAI-compatible proxy for LLM benchmark turns.""" from __future__ import annotations @@ -11,32 +11,33 @@ from typing import Any import structlog -from fastapi import APIRouter, BackgroundTasks, Depends, Header, HTTPException +from fastapi import APIRouter, BackgroundTasks, Depends, Header, HTTPException, Path from psycopg_pool import AsyncConnectionPool from pydantic import BaseModel, Field from starlette.responses import JSONResponse from coval_bench.api.deps import ( bearer_token, - get_phonely_client, get_pool, get_settings, + get_turn_client, secret_matches, ) from coval_bench.config import Settings from coval_bench.db.llm_turns import insert_turn -from coval_bench.llm.phonely import MODEL, PROVIDER, PhonelyClient, PhonelyError, TurnResult +from coval_bench.llm.benchmark import LLM_MODELS +from coval_bench.llm.turn import TurnClient, TurnError, TurnResult -logger = structlog.get_logger("coval_bench.api.llm_phonely") +logger = structlog.get_logger("coval_bench.api.llm_proxy") -router = APIRouter(prefix="/llm/phonely", tags=["llm-phonely"]) +router = APIRouter(prefix="/llm/{provider}", tags=["llm-proxy"]) _MESSAGE_KEYS = frozenset({"role", "content", "name", "tool_calls", "tool_call_id"}) _TURN_TIMEOUT_S = 150.0 -# Only model (the Phonely callId) and messages reach Phonely; its agent owns tools and -# sampling, so other OpenAI request fields are dropped. +# Only model (the provider's session id) and messages reach the provider; its agent +# owns tools and sampling, so other OpenAI request fields are dropped. class ChatRequest(BaseModel): model: str = Field(min_length=1) messages: list[dict[str, Any]] @@ -69,6 +70,7 @@ def _messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: async def _record_turn( pool: AsyncConnectionPool[Any], *, + provider: str, simulation_id: str, turn_index: int, result: TurnResult, @@ -78,8 +80,8 @@ async def _record_turn( pool, simulation_id=simulation_id, turn_index=turn_index, - provider=PROVIDER, - model=MODEL, + provider=provider, + model=LLM_MODELS[provider], ttft_ms=result.ttft_ms, total_ms=result.total_ms, output_tokens=result.output_tokens, @@ -122,13 +124,14 @@ def _completion(call_id: str, result: TurnResult) -> dict[str, Any]: @router.post("/session", dependencies=[Depends(require_proxy_secret)]) async def create_session( - client: PhonelyClient = Depends(get_phonely_client), + provider: str = Path(), + client: TurnClient = Depends(get_turn_client), ) -> dict[str, str | None]: try: session = await client.create_session() - except PhonelyError as exc: - logger.warning("phonely_session_failed", error=str(exc)) - raise HTTPException(502, "Phonely session creation failed") from exc + except TurnError as exc: + logger.warning("llm_session_failed", provider=provider, error=str(exc)) + raise HTTPException(502, f"{provider} session creation failed") from exc return {"sessionId": session.call_id, "expiresAt": session.expires_at} @@ -136,7 +139,8 @@ async def create_session( async def chat( body: ChatRequest, background: BackgroundTasks, - client: PhonelyClient = Depends(get_phonely_client), + provider: str = Path(), + client: TurnClient = Depends(get_turn_client), pool: AsyncConnectionPool[Any] = Depends(get_pool), ) -> JSONResponse: if body.stream: @@ -147,22 +151,23 @@ async def chat( async with asyncio.timeout(_TURN_TIMEOUT_S): result = await client.stream_turn(body.model, messages) except TimeoutError as exc: - logger.warning("phonely_turn_timed_out", turn_index=turn_index) - raise HTTPException(504, "Phonely completion timed out") from exc - except PhonelyError as exc: - logger.warning("phonely_turn_failed", turn_index=turn_index, error=str(exc)) - raise HTTPException(502, "Phonely completion failed") from exc + logger.warning("llm_turn_timed_out", provider=provider, turn_index=turn_index) + raise HTTPException(504, f"{provider} completion timed out") from exc + except TurnError as exc: + logger.warning("llm_turn_failed", provider=provider, turn_index=turn_index, error=str(exc)) + raise HTTPException(502, f"{provider} completion failed") from exc if body.simulation_id: background.add_task( _record_turn, pool, + provider=provider, simulation_id=body.simulation_id, turn_index=turn_index, result=result, ) else: - logger.warning("llm_turn_unattributed", turn_index=turn_index) + logger.warning("llm_turn_unattributed", provider=provider, turn_index=turn_index) return JSONResponse( _completion(body.model, result), headers={ diff --git a/runner/src/coval_bench/llm/benchmark.py b/runner/src/coval_bench/llm/benchmark.py index 171eef27..0be1ce73 100644 --- a/runner/src/coval_bench/llm/benchmark.py +++ b/runner/src/coval_bench/llm/benchmark.py @@ -8,6 +8,7 @@ from collections.abc import Iterable from typing import Any +LLM_MODELS = {"phonely": "phonely-agent"} DEFAULT_PERSONA_ID = "PN3xgmsqeLDjsNNEA2e55e" ITERATION_COUNT = 1 TEMPLATE_MANAGED = ("agent_ids", "persona_ids", "test_set_ids", "metric_ids", "iteration_count") diff --git a/runner/src/coval_bench/llm/coval_agent.py b/runner/src/coval_bench/llm/coval_agent.py index 0f851bf5..07e02e42 100644 --- a/runner/src/coval_bench/llm/coval_agent.py +++ b/runner/src/coval_bench/llm/coval_agent.py @@ -20,11 +20,8 @@ from coval_bench.platform_assets import COVAL_API_BASE, COVAL_API_KEY, CovalClient, SyncError, plan from coval_bench.variants.platforms import redact -CUSTOMER_AGENT_ID = "benchmarks-phonely-text" -DISPLAY_NAME = "Benchmarks: Phonely text agent" # The public API rejects MODEL_TYPE_TEXT; CHAT is the HTTP text simulator. MODEL_TYPE = "MODEL_TYPE_CHAT" -RUN_NAME = "benchmarks-phonely-text-daily" SCHEDULE_EXPRESSION = "cron(0 13 * * ? *)" SCHEDULE_TIMEZONE = "UTC" INPUT_TEMPLATE = ( @@ -36,13 +33,28 @@ class CovalTextAgentDefinition(BaseModel, frozen=True): + provider: str proxy_url: str proxy_secret: SecretStr test_set_id: str instruction_metric_id: str + @property + def customer_agent_id(self) -> str: + return f"benchmarks-{self.provider}-text" + + @property + def display_name(self) -> str: + return f"Benchmarks: {self.provider.capitalize()} text agent" + + @property + def run_name(self) -> str: + return f"benchmarks-{self.provider}-text-daily" + @classmethod - def from_settings(cls, settings: Settings, *, test_set_id: str | None = None) -> Self: + def from_settings( + cls, provider: str, settings: Settings, *, test_set_id: str | None = None + ) -> Self: proxy_url = settings.llm_proxy_public_url proxy_secret = settings.llm_proxy_secret dental = test_set_id or settings.coval_s2s_dental_test_set_id @@ -60,6 +72,7 @@ def from_settings(cls, settings: Settings, *, test_set_id: str | None = None) -> if missing or proxy_url is None or proxy_secret is None or not dental or not metric: raise SyncError(f"sync-llm needs {', '.join(missing)} set") return cls( + provider=provider, proxy_url=proxy_url.rstrip("/"), proxy_secret=proxy_secret, test_set_id=dental, @@ -68,12 +81,12 @@ def from_settings(cls, settings: Settings, *, test_set_id: str | None = None) -> def agent_body(self) -> dict[str, Any]: return { - "display_name": DISPLAY_NAME, - "customer_agent_id": CUSTOMER_AGENT_ID, + "display_name": self.display_name, + "customer_agent_id": self.customer_agent_id, "model_type": MODEL_TYPE, "metadata": { - "chat_endpoint": f"{self.proxy_url}/llm/phonely/chat", - "initialization_endpoint": f"{self.proxy_url}/llm/phonely/session", + "chat_endpoint": f"{self.proxy_url}/llm/{self.provider}/chat", + "initialization_endpoint": f"{self.proxy_url}/llm/{self.provider}/session", "initialization_payload": "{}", "authorization_header": f"Bearer {self.proxy_secret.get_secret_value()}", "input_template": INPUT_TEMPLATE, @@ -101,13 +114,13 @@ def redacted_body(self) -> dict[str, Any]: def run_template_body(self, agent_id: str) -> dict[str, Any]: return benchmark.run_template_body( - RUN_NAME, agent_id, self.test_set_id, self.instruction_metric_id + self.run_name, agent_id, self.test_set_id, self.instruction_metric_id ) -def scheduled_run_body(run_template_id: str) -> dict[str, Any]: +def scheduled_run_body(run_name: str, run_template_id: str) -> dict[str, Any]: return { - "display_name": RUN_NAME, + "display_name": run_name, "run_template_id": run_template_id, "schedule_expression": SCHEDULE_EXPRESSION, "schedule_timezone": SCHEDULE_TIMEZONE, @@ -169,7 +182,7 @@ def sync( """Find-or-create the agent, its test-set link, run template, and schedule.""" result = SyncResult() wanted = definition.agent_body() - live = client.find_agent(CUSTOMER_AGENT_ID) + live = client.find_agent(definition.customer_agent_id) if live is None: result.actions.append("agent: create") if dry_run: @@ -200,7 +213,7 @@ def sync( if not dry_run: client.add_test_set_agents(definition.test_set_id, [result.agent_id]) - template = client.find_run_template(RUN_NAME) + template = client.find_run_template(definition.run_name) if template is None: result.actions.append("run template: create") if dry_run: @@ -224,7 +237,7 @@ def sync( if client.find_scheduled_run(template_id) is None: result.actions.append("scheduled run: create") if not dry_run: - client.create_scheduled_run(scheduled_run_body(template_id)) + client.create_scheduled_run(scheduled_run_body(definition.run_name, template_id)) else: result.actions.append("scheduled run: exists") return result @@ -249,27 +262,47 @@ def sync_llm(dry_run: bool, test_set_id: str | None, coval_api_base: str) -> Non configure_logging(level=settings.log_level) run_logger = structlog.get_logger("coval_bench.llm.coval_agent") try: - definition = CovalTextAgentDefinition.from_settings(settings, test_set_id=test_set_id) + definitions = [ + CovalTextAgentDefinition.from_settings(provider, settings, test_set_id=test_set_id) + for provider in benchmark.LLM_MODELS + ] if dry_run: - click.echo(json.dumps(definition.redacted_body(), indent=2, sort_keys=True)) + for definition in definitions: + click.echo(json.dumps(definition.redacted_body(), indent=2, sort_keys=True)) with CovalTextClient(COVAL_API_KEY.resolve(), coval_api_base) as client: - result = sync(client, definition, dry_run=dry_run) + results = { + definition.provider: sync(client, definition, dry_run=dry_run) + for definition in definitions + } except (SyncError, RuntimeError, httpx.HTTPError) as exc: if not dry_run: run_logger.error("RUN_FAILED", error=str(exc), exc_info=exc) raise click.ClickException(str(exc)) from exc - for action in result.actions: - click.echo(action) - click.echo(f"COVAL_LLM_PHONELY_AGENT_ID={result.agent_id or ''}") - if not dry_run: + for provider, result in results.items(): + for action in result.actions: + click.echo(f"{provider} {action}") + click.echo( + f"COVAL_LLM_{provider.upper()}_AGENT_ID={result.agent_id or ''}" + ) + if dry_run: + return + fetchable: dict[str, str] = {} + for provider, result in results.items(): if "scheduled run: create" in result.actions: - run_logger.info("llm_sync_fetch_deferred", reason="scheduled_run_created") - else: - from coval_bench.registries.benchmarks import Benchmark - from coval_bench.s2s.fetch_v2v import _run_fetch - - fetch_settings = settings.model_copy( - update={"coval_llm_phonely_agent_id": result.agent_id} + run_logger.info( + "llm_sync_fetch_deferred", provider=provider, reason="scheduled_run_created" ) - _run_fetch(Benchmark.LLM, (), 720, 100, settings=fetch_settings) - run_logger.info("llm_sync_completed", agent_id=result.agent_id, actions=result.actions) + else: + fetchable[f"coval_llm_{provider}_agent_id"] = result.agent_id + if fetchable: + from coval_bench.registries.benchmarks import Benchmark + from coval_bench.s2s.fetch_v2v import _run_fetch + + _run_fetch(Benchmark.LLM, (), 720, 100, settings=settings.model_copy(update=fetchable)) + for provider, result in results.items(): + run_logger.info( + "llm_sync_completed", + provider=provider, + agent_id=result.agent_id, + actions=result.actions, + ) diff --git a/runner/src/coval_bench/llm/phonely.py b/runner/src/coval_bench/llm/phonely.py index daa88c69..1fb576e4 100644 --- a/runner/src/coval_bench/llm/phonely.py +++ b/runner/src/coval_bench/llm/phonely.py @@ -7,21 +7,19 @@ import json import time -from dataclasses import dataclass from typing import Any import httpx -# The (provider, model) identity seeded into benchmarks_v2.models by migration 0025. -PROVIDER = "phonely" -MODEL = "phonely-agent" +from coval_bench.llm.turn import Session as PhonelySession +from coval_bench.llm.turn import TurnError, TurnResult _SESSION_TIMEOUT = httpx.Timeout(30.0) # Turns are seconds apart; a 5s keepalive would put a TLS handshake inside most TTFTs. _LIMITS = httpx.Limits(max_connections=20, max_keepalive_connections=8, keepalive_expiry=300.0) -class PhonelyError(Exception): +class PhonelyError(TurnError): """Base class for failures returned by the Phonely API.""" @@ -37,22 +35,6 @@ class PhonelyUpstreamError(PhonelyError): """Phonely failed to produce a usable completion.""" -@dataclass(frozen=True) -class PhonelySession: - call_id: str - expires_at: str | None - - -@dataclass(frozen=True) -class TurnResult: - content: str - tool_calls: tuple[dict[str, Any], ...] - finish_reason: str - ttft_ms: float - total_ms: float - output_tokens: int | None - - class TurnAccumulator: """Reassemble OpenAI SSE chunks while measuring the first meaningful delta.""" diff --git a/runner/src/coval_bench/llm/turn.py b/runner/src/coval_bench/llm/turn.py new file mode 100644 index 00000000..d8dfe102 --- /dev/null +++ b/runner/src/coval_bench/llm/turn.py @@ -0,0 +1,37 @@ +# Copyright 2026 The Coval Benchmarks Authors +# SPDX-License-Identifier: Apache-2.0 + +"""The turn contract every LLM proxied for Coval implements.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Protocol + + +class TurnError(Exception): + """A provider failed to open a session or produce a usable completion.""" + + +@dataclass(frozen=True) +class Session: + call_id: str + expires_at: str | None + + +@dataclass(frozen=True) +class TurnResult: + content: str + tool_calls: tuple[dict[str, Any], ...] + finish_reason: str + ttft_ms: float + total_ms: float + output_tokens: int | None + + +class TurnClient(Protocol): + async def create_session(self) -> Session: ... + + async def stream_turn(self, call_id: str, messages: list[dict[str, Any]]) -> TurnResult: ... + + async def aclose(self) -> None: ... diff --git a/runner/src/coval_bench/s2s/fetch_v2v.py b/runner/src/coval_bench/s2s/fetch_v2v.py index 7545cae6..974b491e 100644 --- a/runner/src/coval_bench/s2s/fetch_v2v.py +++ b/runner/src/coval_bench/s2s/fetch_v2v.py @@ -27,8 +27,7 @@ from coval_bench.db.conn import lifespan_pool from coval_bench.db.models import MetricExecutor, Result, ResultStatus, RunStatus from coval_bench.db.writer import RunWriter -from coval_bench.llm.phonely import MODEL as PHONELY_MODEL -from coval_bench.llm.phonely import PROVIDER as PHONELY_PROVIDER +from coval_bench.llm.benchmark import LLM_MODELS from coval_bench.registries import METRIC_SPECS, Metric from coval_bench.registries.benchmarks import Benchmark from coval_bench.s2s.conditions import ( @@ -132,14 +131,17 @@ class CovalRun: ), # The same dental set driven over text through the LLM proxy; TTFT comes from # the proxy's own turn log rather than from Coval. - AgentSpec( - agent_id_attr="coval_llm_phonely_agent_id", - provider=PHONELY_PROVIDER, - model=PHONELY_MODEL, - test_set_id_attr="coval_s2s_dental_test_set_id", - family=FAMILY_LLM_DENTAL, - publish_samples=False, - benchmark=Benchmark.LLM, + *( + AgentSpec( + agent_id_attr=f"coval_llm_{provider}_agent_id", + provider=provider, + model=model, + test_set_id_attr="coval_s2s_dental_test_set_id", + family=FAMILY_LLM_DENTAL, + publish_samples=False, + benchmark=Benchmark.LLM, + ) + for provider, model in LLM_MODELS.items() ), ) diff --git a/runner/tests/api/test_llm_phonely.py b/runner/tests/api/test_llm_proxy.py similarity index 93% rename from runner/tests/api/test_llm_phonely.py rename to runner/tests/api/test_llm_proxy.py index c9e60282..e0ef8193 100644 --- a/runner/tests/api/test_llm_phonely.py +++ b/runner/tests/api/test_llm_proxy.py @@ -1,7 +1,7 @@ # Copyright 2026 The Coval Benchmarks Authors # SPDX-License-Identifier: Apache-2.0 -"""The authenticated Phonely LLM proxy.""" +"""The authenticated LLM proxy, exercised through the Phonely provider.""" from __future__ import annotations @@ -57,7 +57,7 @@ async def bind_phonely(app: FastAPI) -> AsyncIterator[Callable[[Handler], None]] "https://phonely.test", transport=httpx.MockTransport(lambda request: handlers[-1](request)), ) - app.state.phonely_client = client + app.state.llm_clients = {"phonely": client} yield handlers.append await client.aclose() @@ -78,8 +78,8 @@ async def _turn_rows(postgresql: Any) -> list[dict[str, Any]]: async def test_proxy_auth_configuration_and_route_location( client: AsyncClient, app: FastAPI ) -> None: - configured_client = app.state.phonely_client - assert isinstance(configured_client, PhonelyClient) + configured_clients = app.state.llm_clients + assert isinstance(configured_clients["phonely"], PhonelyClient) assert (await client.post("/llm/phonely/session", json={})).status_code == 401 assert ( await client.post( @@ -87,13 +87,14 @@ async def test_proxy_auth_configuration_and_route_location( ) ).status_code == 401 assert (await client.post("/v1/llm/phonely/session", json={}, headers=AUTH)).status_code == 404 + assert (await client.post("/llm/unknown/session", json={}, headers=AUTH)).status_code == 404 - app.state.phonely_client = None + app.state.llm_clients = {} assert (await client.post("/llm/phonely/session", json={}, headers=AUTH)).status_code == 503 lowercase = {"Authorization": f"bearer {LLM_PROXY_KEY}"} lowercase_response = await client.post("/llm/phonely/session", json={}, headers=lowercase) assert lowercase_response.status_code == 503 - app.state.phonely_client = configured_client + app.state.llm_clients = configured_clients settings = app.state.settings app.state.settings = settings.model_copy(update={"llm_proxy_secret": SecretStr("")}) empty = {"Authorization": "Bearer "} @@ -239,7 +240,7 @@ async def test_a_stalled_turn_returns_504( bind_phonely: Callable[[Handler], None], monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr("coval_bench.api.routers.llm_phonely._TURN_TIMEOUT_S", 0.2) + monkeypatch.setattr("coval_bench.api.routers.llm_proxy._TURN_TIMEOUT_S", 0.2) bind_phonely(lambda _request: httpx.Response(200, stream=_StalledStream())) response = await client.post( "/llm/phonely/chat", headers=AUTH, json={"model": "call-1", "messages": []} @@ -254,11 +255,11 @@ async def test_failed_timing_insert_does_not_fail_the_turn( ) -> None: bind_phonely(lambda _request: httpx.Response(200, content=_sse({"content": "Hi"}))) monkeypatch.setattr( - "coval_bench.api.routers.llm_phonely.insert_turn", + "coval_bench.api.routers.llm_proxy.insert_turn", AsyncMock(side_effect=RuntimeError("database unavailable")), ) logger = MagicMock() - monkeypatch.setattr("coval_bench.api.routers.llm_phonely.logger", logger) + monkeypatch.setattr("coval_bench.api.routers.llm_proxy.logger", logger) response = await client.post( "/llm/phonely/chat", diff --git a/runner/tests/unit/test_coval_agent_sync.py b/runner/tests/unit/test_coval_agent_sync.py index 0f3f18bc..c5d52178 100644 --- a/runner/tests/unit/test_coval_agent_sync.py +++ b/runner/tests/unit/test_coval_agent_sync.py @@ -17,8 +17,6 @@ from coval_bench.config import Settings from coval_bench.llm import benchmark, coval_agent from coval_bench.llm.coval_agent import ( - CUSTOMER_AGENT_ID, - RUN_NAME, CovalTextAgentDefinition, CovalTextClient, sync, @@ -28,6 +26,7 @@ SECRET = "proxy-secret-value" # noqa: S105 DEFINITION = CovalTextAgentDefinition( + provider="phonely", proxy_url="https://api.example.com", proxy_secret=SecretStr(SECRET), test_set_id="TSDENTAL", @@ -92,7 +91,8 @@ def handler(request: httpx.Request) -> httpx.Response: def test_body_renders_the_proxy_contract_and_survives_covals_substitution() -> None: body = DEFINITION.agent_body() - assert body["customer_agent_id"] == CUSTOMER_AGENT_ID + assert body["customer_agent_id"] == "benchmarks-phonely-text" + assert body["display_name"] == "Benchmarks: Phonely text agent" assert body["model_type"] == "MODEL_TYPE_CHAT" assert body["metadata"]["chat_endpoint"] == "https://api.example.com/llm/phonely/chat" assert body["metadata"]["authorization_header"] == f"Bearer {SECRET}" @@ -113,9 +113,11 @@ def test_body_renders_the_proxy_contract_and_survives_covals_substitution() -> N def test_from_settings_names_every_missing_setting() -> None: with pytest.raises(SyncError, match="llm_proxy_public_url, llm_proxy_secret"): CovalTextAgentDefinition.from_settings( - Settings(coval_s2s_dental_test_set_id="T", coval_s2s_instruction_metric_id="M") + "phonely", + Settings(coval_s2s_dental_test_set_id="T", coval_s2s_instruction_metric_id="M"), ) definition = CovalTextAgentDefinition.from_settings( + "phonely", Settings( llm_proxy_public_url="https://api.example.com/", llm_proxy_secret=SecretStr(SECRET), @@ -205,7 +207,7 @@ def test_sync_looks_up_by_customer_id_filter_and_never_adopts_a_name_only_match( with _client(state) as client: result = sync(client, DEFINITION) assert state["filters"] == [ - {"filter": f'customer_agent_id="{CUSTOMER_AGENT_ID}"', "page_size": "1"} + {"filter": 'customer_agent_id="benchmarks-phonely-text"', "page_size": "1"} ] assert result.agent_id == "A" * 22 assert state["writes"][0][0] == "/agents" @@ -269,7 +271,7 @@ def test_cli_syncs_completed_runs_into_the_database(monkeypatch: pytest.MonkeyPa state = _state( agents=[{**DEFINITION.agent_body(), "id": agent_id}], test_set_agents=[{"id": agent_id}], - run_templates=[{"id": template_id, "display_name": RUN_NAME}], + run_templates=[{"id": template_id, "display_name": "benchmarks-phonely-text-daily"}], scheduled_runs=[{"id": "S" * 22, "run_template_id": template_id}], ) settings = Settings(