diff --git a/runner/src/coval_bench/llm/benchmark.py b/runner/src/coval_bench/llm/benchmark.py new file mode 100644 index 00000000..171eef27 --- /dev/null +++ b/runner/src/coval_bench/llm/benchmark.py @@ -0,0 +1,43 @@ +# Copyright 2026 The Coval Benchmarks Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Coval run settings shared by every LLM benchmark agent.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +DEFAULT_PERSONA_ID = "PN3xgmsqeLDjsNNEA2e55e" +ITERATION_COUNT = 1 +TEMPLATE_MANAGED = ("agent_ids", "persona_ids", "test_set_ids", "metric_ids", "iteration_count") +_TEMPLATE_PATCH_KEYS = { + "agent_ids": "agent_id", + "persona_ids": "persona_id", + "test_set_ids": "test_set_id", +} + + +def run_template_body( + display_name: str, agent_id: str, test_set_id: str, metric_id: str +) -> dict[str, Any]: + return { + "display_name": display_name, + "agent_ids": [agent_id], + "persona_ids": [DEFAULT_PERSONA_ID], + "test_set_ids": [test_set_id], + "metric_ids": [metric_id], + "iteration_count": ITERATION_COUNT, + } + + +def template_patch_body(wanted: dict[str, Any], paths: Iterable[str]) -> dict[str, Any]: + body: dict[str, Any] = {} + for path in paths: + singular = _TEMPLATE_PATCH_KEYS.get(path) + if singular is None: + body[path] = wanted[path] + continue + (only,) = wanted[path] + body[singular] = only + return body diff --git a/runner/src/coval_bench/llm/coval_agent.py b/runner/src/coval_bench/llm/coval_agent.py index 462c2d02..0f851bf5 100644 --- a/runner/src/coval_bench/llm/coval_agent.py +++ b/runner/src/coval_bench/llm/coval_agent.py @@ -15,6 +15,7 @@ from pydantic import BaseModel, SecretStr from coval_bench.config import Settings, get_settings +from coval_bench.llm import benchmark from coval_bench.logging import configure_logging from coval_bench.platform_assets import COVAL_API_BASE, COVAL_API_KEY, CovalClient, SyncError, plan from coval_bench.variants.platforms import redact @@ -24,7 +25,6 @@ # The public API rejects MODEL_TYPE_TEXT; CHAT is the HTTP text simulator. MODEL_TYPE = "MODEL_TYPE_CHAT" RUN_NAME = "benchmarks-phonely-text-daily" -CLEAN_DENTAL_PERSONAS = ("PN3xgmsqeLDjsNNEA2e55e", "9ATy64zKXxSUaVWb5YnQtd") SCHEDULE_EXPRESSION = "cron(0 13 * * ? *)" SCHEDULE_TIMEZONE = "UTC" INPUT_TEMPLATE = ( @@ -100,14 +100,9 @@ def redacted_body(self) -> dict[str, Any]: return redacted def run_template_body(self, agent_id: str) -> dict[str, Any]: - return { - "display_name": RUN_NAME, - "agent_ids": [agent_id], - "persona_ids": list(CLEAN_DENTAL_PERSONAS), - "test_set_ids": [self.test_set_id], - "metric_ids": [self.instruction_metric_id], - "iteration_count": 1, - } + return benchmark.run_template_body( + RUN_NAME, agent_id, self.test_set_id, self.instruction_metric_id + ) def scheduled_run_body(run_template_id: str) -> dict[str, Any]: @@ -145,6 +140,11 @@ def create_run_template(self, body: dict[str, Any]) -> dict[str, Any]: template = payload.get("run_template") return template if isinstance(template, dict) else payload + def update_run_template(self, template_id: str, body: dict[str, Any]) -> dict[str, Any]: + payload = self._request("PATCH", f"/run-templates/{template_id}", body) + template = payload.get("run_template") + return template if isinstance(template, dict) else payload + def find_scheduled_run(self, run_template_id: str) -> dict[str, Any] | None: for scheduled in self._pages("/scheduled-runs", "scheduled_runs"): if scheduled.get("run_template_id") == run_template_id: @@ -208,7 +208,17 @@ def sync( return result template = client.create_run_template(definition.run_template_body(result.agent_id)) else: - result.actions.append("run template: exists") + wanted_template = definition.run_template_body(result.agent_id) + drift = plan(template, {path: wanted_template[path] for path in benchmark.TEMPLATE_MANAGED}) + if drift.update: + result.actions.append(f"run template: patch {sorted(drift.update)}") + if not dry_run: + client.update_run_template( + str(template["id"]), + benchmark.template_patch_body(wanted_template, drift.update), + ) + else: + result.actions.append("run template: unchanged") template_id = str(template["id"]) if client.find_scheduled_run(template_id) is None: diff --git a/runner/tests/unit/test_coval_agent_sync.py b/runner/tests/unit/test_coval_agent_sync.py index 18cd2445..0f3f18bc 100644 --- a/runner/tests/unit/test_coval_agent_sync.py +++ b/runner/tests/unit/test_coval_agent_sync.py @@ -15,7 +15,7 @@ from pydantic import SecretStr from coval_bench.config import Settings -from coval_bench.llm import coval_agent +from coval_bench.llm import benchmark, coval_agent from coval_bench.llm.coval_agent import ( CUSTOMER_AGENT_ID, RUN_NAME, @@ -79,6 +79,8 @@ def handler(request: httpx.Request) -> httpx.Response: created = {**body, "id": "T" * 22} state["run_templates"].append(created) return httpx.Response(200, json={"run_template": created}) + if path.startswith("/run-templates/"): + return httpx.Response(200, json={"run_template": {**state["run_templates"][0], **body}}) if path == "/scheduled-runs": return httpx.Response(200, json={"scheduled_run": {**body, "id": "S" * 22}}) return httpx.Response( @@ -144,7 +146,7 @@ def test_sync_creates_everything_when_absent() -> None: ] template = state["writes"][2][1] assert template["agent_ids"] == ["A" * 22] - assert template["persona_ids"] == list(coval_agent.CLEAN_DENTAL_PERSONAS) + assert template["persona_ids"] == [benchmark.DEFAULT_PERSONA_ID] assert template["test_set_ids"] == ["TSDENTAL"] assert template["metric_ids"] == ["M" * 22] assert template["iteration_count"] == 1 @@ -158,7 +160,7 @@ def test_sync_patches_drifted_metadata_wholesale_and_leaves_the_rest() -> None: state = _state( agents=[live], test_set_agents=[{"id": "A"}], - run_templates=[{"id": "T", "display_name": RUN_NAME}], + run_templates=[{**DEFINITION.run_template_body("A"), "id": "T"}], scheduled_runs=[{"id": "S", "run_template_id": "T"}], ) with _client(state) as client: @@ -167,12 +169,36 @@ def test_sync_patches_drifted_metadata_wholesale_and_leaves_the_rest() -> None: assert result.actions == [ "agent: patch ['metadata']", "test set: attached", - "run template: exists", + "run template: unchanged", "scheduled run: exists", ] assert state["writes"] == [("/agents/A", {"metadata": DEFINITION.agent_body()["metadata"]})] +def test_sync_patches_only_the_drifted_template_fields() -> None: + live_template = { + **DEFINITION.run_template_body("A"), + "id": "T", + "persona_ids": [benchmark.DEFAULT_PERSONA_ID, "9ATy64zKXxSUaVWb5YnQtd"], + "concurrency": 1, + } + state = _state( + agents=[{**DEFINITION.agent_body(), "id": "A"}], + test_set_agents=[{"id": "A"}], + run_templates=[live_template], + scheduled_runs=[{"id": "S", "run_template_id": "T"}], + ) + with _client(state) as client: + assert ( + sync(client, DEFINITION, dry_run=True).actions[2] + == "run template: patch ['persona_ids']" + ) + assert state["writes"] == [] + sync(client, DEFINITION) + + assert state["writes"] == [("/run-templates/T", {"persona_id": benchmark.DEFAULT_PERSONA_ID})] + + def test_sync_looks_up_by_customer_id_filter_and_never_adopts_a_name_only_match() -> None: name_only = {**DEFINITION.agent_body(), "id": "B", "customer_agent_id": ""} state = _state(agents=[name_only])