From 8a371759bd68d89340ca83e6dad35deb7d3f3946 Mon Sep 17 00:00:00 2001 From: Cale Smith Date: Tue, 8 Sep 2026 11:00:23 -0700 Subject: [PATCH] [BENCH-825] Run LLM benchmarks with a single default persona Coval fans out one run per persona over the whole test set, so the two dental personas doubled the daily Phonely load to 100 conversations. Move the run-template defaults into a shared LLM benchmark module with one persona, and make sync-llm reconcile the live template so the change reaches the template Coval already created. --- runner/src/coval_bench/llm/benchmark.py | 43 ++++++++++++++++++++++ runner/src/coval_bench/llm/coval_agent.py | 30 ++++++++++----- runner/tests/unit/test_coval_agent_sync.py | 34 +++++++++++++++-- 3 files changed, 93 insertions(+), 14 deletions(-) create mode 100644 runner/src/coval_bench/llm/benchmark.py diff --git a/runner/src/coval_bench/llm/benchmark.py b/runner/src/coval_bench/llm/benchmark.py new file mode 100644 index 00000000..171eef27 --- /dev/null +++ b/runner/src/coval_bench/llm/benchmark.py @@ -0,0 +1,43 @@ +# Copyright 2026 The Coval Benchmarks Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Coval run settings shared by every LLM benchmark agent.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +DEFAULT_PERSONA_ID = "PN3xgmsqeLDjsNNEA2e55e" +ITERATION_COUNT = 1 +TEMPLATE_MANAGED = ("agent_ids", "persona_ids", "test_set_ids", "metric_ids", "iteration_count") +_TEMPLATE_PATCH_KEYS = { + "agent_ids": "agent_id", + "persona_ids": "persona_id", + "test_set_ids": "test_set_id", +} + + +def run_template_body( + display_name: str, agent_id: str, test_set_id: str, metric_id: str +) -> dict[str, Any]: + return { + "display_name": display_name, + "agent_ids": [agent_id], + "persona_ids": [DEFAULT_PERSONA_ID], + "test_set_ids": [test_set_id], + "metric_ids": [metric_id], + "iteration_count": ITERATION_COUNT, + } + + +def template_patch_body(wanted: dict[str, Any], paths: Iterable[str]) -> dict[str, Any]: + body: dict[str, Any] = {} + for path in paths: + singular = _TEMPLATE_PATCH_KEYS.get(path) + if singular is None: + body[path] = wanted[path] + continue + (only,) = wanted[path] + body[singular] = only + return body diff --git a/runner/src/coval_bench/llm/coval_agent.py b/runner/src/coval_bench/llm/coval_agent.py index 462c2d02..0f851bf5 100644 --- a/runner/src/coval_bench/llm/coval_agent.py +++ b/runner/src/coval_bench/llm/coval_agent.py @@ -15,6 +15,7 @@ from pydantic import BaseModel, SecretStr from coval_bench.config import Settings, get_settings +from coval_bench.llm import benchmark from coval_bench.logging import configure_logging from coval_bench.platform_assets import COVAL_API_BASE, COVAL_API_KEY, CovalClient, SyncError, plan from coval_bench.variants.platforms import redact @@ -24,7 +25,6 @@ # The public API rejects MODEL_TYPE_TEXT; CHAT is the HTTP text simulator. MODEL_TYPE = "MODEL_TYPE_CHAT" RUN_NAME = "benchmarks-phonely-text-daily" -CLEAN_DENTAL_PERSONAS = ("PN3xgmsqeLDjsNNEA2e55e", "9ATy64zKXxSUaVWb5YnQtd") SCHEDULE_EXPRESSION = "cron(0 13 * * ? *)" SCHEDULE_TIMEZONE = "UTC" INPUT_TEMPLATE = ( @@ -100,14 +100,9 @@ def redacted_body(self) -> dict[str, Any]: return redacted def run_template_body(self, agent_id: str) -> dict[str, Any]: - return { - "display_name": RUN_NAME, - "agent_ids": [agent_id], - "persona_ids": list(CLEAN_DENTAL_PERSONAS), - "test_set_ids": [self.test_set_id], - "metric_ids": [self.instruction_metric_id], - "iteration_count": 1, - } + return benchmark.run_template_body( + RUN_NAME, agent_id, self.test_set_id, self.instruction_metric_id + ) def scheduled_run_body(run_template_id: str) -> dict[str, Any]: @@ -145,6 +140,11 @@ def create_run_template(self, body: dict[str, Any]) -> dict[str, Any]: template = payload.get("run_template") return template if isinstance(template, dict) else payload + def update_run_template(self, template_id: str, body: dict[str, Any]) -> dict[str, Any]: + payload = self._request("PATCH", f"/run-templates/{template_id}", body) + template = payload.get("run_template") + return template if isinstance(template, dict) else payload + def find_scheduled_run(self, run_template_id: str) -> dict[str, Any] | None: for scheduled in self._pages("/scheduled-runs", "scheduled_runs"): if scheduled.get("run_template_id") == run_template_id: @@ -208,7 +208,17 @@ def sync( return result template = client.create_run_template(definition.run_template_body(result.agent_id)) else: - result.actions.append("run template: exists") + wanted_template = definition.run_template_body(result.agent_id) + drift = plan(template, {path: wanted_template[path] for path in benchmark.TEMPLATE_MANAGED}) + if drift.update: + result.actions.append(f"run template: patch {sorted(drift.update)}") + if not dry_run: + client.update_run_template( + str(template["id"]), + benchmark.template_patch_body(wanted_template, drift.update), + ) + else: + result.actions.append("run template: unchanged") template_id = str(template["id"]) if client.find_scheduled_run(template_id) is None: diff --git a/runner/tests/unit/test_coval_agent_sync.py b/runner/tests/unit/test_coval_agent_sync.py index 18cd2445..0f3f18bc 100644 --- a/runner/tests/unit/test_coval_agent_sync.py +++ b/runner/tests/unit/test_coval_agent_sync.py @@ -15,7 +15,7 @@ from pydantic import SecretStr from coval_bench.config import Settings -from coval_bench.llm import coval_agent +from coval_bench.llm import benchmark, coval_agent from coval_bench.llm.coval_agent import ( CUSTOMER_AGENT_ID, RUN_NAME, @@ -79,6 +79,8 @@ def handler(request: httpx.Request) -> httpx.Response: created = {**body, "id": "T" * 22} state["run_templates"].append(created) return httpx.Response(200, json={"run_template": created}) + if path.startswith("/run-templates/"): + return httpx.Response(200, json={"run_template": {**state["run_templates"][0], **body}}) if path == "/scheduled-runs": return httpx.Response(200, json={"scheduled_run": {**body, "id": "S" * 22}}) return httpx.Response( @@ -144,7 +146,7 @@ def test_sync_creates_everything_when_absent() -> None: ] template = state["writes"][2][1] assert template["agent_ids"] == ["A" * 22] - assert template["persona_ids"] == list(coval_agent.CLEAN_DENTAL_PERSONAS) + assert template["persona_ids"] == [benchmark.DEFAULT_PERSONA_ID] assert template["test_set_ids"] == ["TSDENTAL"] assert template["metric_ids"] == ["M" * 22] assert template["iteration_count"] == 1 @@ -158,7 +160,7 @@ def test_sync_patches_drifted_metadata_wholesale_and_leaves_the_rest() -> None: state = _state( agents=[live], test_set_agents=[{"id": "A"}], - run_templates=[{"id": "T", "display_name": RUN_NAME}], + run_templates=[{**DEFINITION.run_template_body("A"), "id": "T"}], scheduled_runs=[{"id": "S", "run_template_id": "T"}], ) with _client(state) as client: @@ -167,12 +169,36 @@ def test_sync_patches_drifted_metadata_wholesale_and_leaves_the_rest() -> None: assert result.actions == [ "agent: patch ['metadata']", "test set: attached", - "run template: exists", + "run template: unchanged", "scheduled run: exists", ] assert state["writes"] == [("/agents/A", {"metadata": DEFINITION.agent_body()["metadata"]})] +def test_sync_patches_only_the_drifted_template_fields() -> None: + live_template = { + **DEFINITION.run_template_body("A"), + "id": "T", + "persona_ids": [benchmark.DEFAULT_PERSONA_ID, "9ATy64zKXxSUaVWb5YnQtd"], + "concurrency": 1, + } + state = _state( + agents=[{**DEFINITION.agent_body(), "id": "A"}], + test_set_agents=[{"id": "A"}], + run_templates=[live_template], + scheduled_runs=[{"id": "S", "run_template_id": "T"}], + ) + with _client(state) as client: + assert ( + sync(client, DEFINITION, dry_run=True).actions[2] + == "run template: patch ['persona_ids']" + ) + assert state["writes"] == [] + sync(client, DEFINITION) + + assert state["writes"] == [("/run-templates/T", {"persona_id": benchmark.DEFAULT_PERSONA_ID})] + + def test_sync_looks_up_by_customer_id_filter_and_never_adopts_a_name_only_match() -> None: name_only = {**DEFINITION.agent_body(), "id": "B", "customer_agent_id": ""} state = _state(agents=[name_only])