Run LLM benchmarks with a single default persona

runner/src/coval_bench/llm/benchmark.py

@@ -0,0 +1,43 @@
+# Copyright 2026 The Coval Benchmarks Authors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Coval run settings shared by every LLM benchmark agent."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import Any
+
+DEFAULT_PERSONA_ID = "PN3xgmsqeLDjsNNEA2e55e"
+ITERATION_COUNT = 1
+TEMPLATE_MANAGED = ("agent_ids", "persona_ids", "test_set_ids", "metric_ids", "iteration_count")
+_TEMPLATE_PATCH_KEYS = {
+    "agent_ids": "agent_id",
+    "persona_ids": "persona_id",
+    "test_set_ids": "test_set_id",
+}
+
+
+def run_template_body(
+    display_name: str, agent_id: str, test_set_id: str, metric_id: str
+) -> dict[str, Any]:
+    return {
+        "display_name": display_name,
+        "agent_ids": [agent_id],
+        "persona_ids": [DEFAULT_PERSONA_ID],
+        "test_set_ids": [test_set_id],
+        "metric_ids": [metric_id],
+        "iteration_count": ITERATION_COUNT,
+    }
+
+
+def template_patch_body(wanted: dict[str, Any], paths: Iterable[str]) -> dict[str, Any]:
+    body: dict[str, Any] = {}
+    for path in paths:
+        singular = _TEMPLATE_PATCH_KEYS.get(path)
+        if singular is None:
+            body[path] = wanted[path]
+            continue
+        (only,) = wanted[path]
+        body[singular] = only
+    return body

runner/src/coval_bench/llm/coval_agent.py

@@ -15,6 +15,7 @@
 from pydantic import BaseModel, SecretStr
 
 from coval_bench.config import Settings, get_settings
+from coval_bench.llm import benchmark
 from coval_bench.logging import configure_logging
 from coval_bench.platform_assets import COVAL_API_BASE, COVAL_API_KEY, CovalClient, SyncError, plan
 from coval_bench.variants.platforms import redact
@@ -24,7 +25,6 @@
 # The public API rejects MODEL_TYPE_TEXT; CHAT is the HTTP text simulator.
 MODEL_TYPE = "MODEL_TYPE_CHAT"
 RUN_NAME = "benchmarks-phonely-text-daily"
-CLEAN_DENTAL_PERSONAS = ("PN3xgmsqeLDjsNNEA2e55e", "9ATy64zKXxSUaVWb5YnQtd")
 SCHEDULE_EXPRESSION = "cron(0 13 * * ? *)"
 SCHEDULE_TIMEZONE = "UTC"
 INPUT_TEMPLATE = (
@@ -100,14 +100,9 @@ def redacted_body(self) -> dict[str, Any]:
         return redacted
 
     def run_template_body(self, agent_id: str) -> dict[str, Any]:
-        return {
-            "display_name": RUN_NAME,
-            "agent_ids": [agent_id],
-            "persona_ids": list(CLEAN_DENTAL_PERSONAS),
-            "test_set_ids": [self.test_set_id],
-            "metric_ids": [self.instruction_metric_id],
-            "iteration_count": 1,
-        }
+        return benchmark.run_template_body(
+            RUN_NAME, agent_id, self.test_set_id, self.instruction_metric_id
+        )
 
 
 def scheduled_run_body(run_template_id: str) -> dict[str, Any]:
@@ -145,6 +140,11 @@ def create_run_template(self, body: dict[str, Any]) -> dict[str, Any]:
         template = payload.get("run_template")
         return template if isinstance(template, dict) else payload
 
+    def update_run_template(self, template_id: str, body: dict[str, Any]) -> dict[str, Any]:
+        payload = self._request("PATCH", f"/run-templates/{template_id}", body)
+        template = payload.get("run_template")
+        return template if isinstance(template, dict) else payload
+
     def find_scheduled_run(self, run_template_id: str) -> dict[str, Any] | None:
         for scheduled in self._pages("/scheduled-runs", "scheduled_runs"):
             if scheduled.get("run_template_id") == run_template_id:
@@ -208,7 +208,17 @@ def sync(
             return result
         template = client.create_run_template(definition.run_template_body(result.agent_id))
     else:
-        result.actions.append("run template: exists")
+        wanted_template = definition.run_template_body(result.agent_id)
+        drift = plan(template, {path: wanted_template[path] for path in benchmark.TEMPLATE_MANAGED})
+        if drift.update:
+            result.actions.append(f"run template: patch {sorted(drift.update)}")
+            if not dry_run:
+                client.update_run_template(
+                    str(template["id"]),
+                    benchmark.template_patch_body(wanted_template, drift.update),
+                )
+        else:
+            result.actions.append("run template: unchanged")
     template_id = str(template["id"])
 
     if client.find_scheduled_run(template_id) is None:

runner/tests/unit/test_coval_agent_sync.py

@@ -15,7 +15,7 @@
 from pydantic import SecretStr
 
 from coval_bench.config import Settings
-from coval_bench.llm import coval_agent
+from coval_bench.llm import benchmark, coval_agent
 from coval_bench.llm.coval_agent import (
     CUSTOMER_AGENT_ID,
     RUN_NAME,
@@ -79,6 +79,8 @@ def handler(request: httpx.Request) -> httpx.Response:
             created = {**body, "id": "T" * 22}
             state["run_templates"].append(created)
             return httpx.Response(200, json={"run_template": created})
+        if path.startswith("/run-templates/"):
+            return httpx.Response(200, json={"run_template": {**state["run_templates"][0], **body}})
         if path == "/scheduled-runs":
             return httpx.Response(200, json={"scheduled_run": {**body, "id": "S" * 22}})
         return httpx.Response(
@@ -144,7 +146,7 @@ def test_sync_creates_everything_when_absent() -> None:
     ]
     template = state["writes"][2][1]
     assert template["agent_ids"] == ["A" * 22]
-    assert template["persona_ids"] == list(coval_agent.CLEAN_DENTAL_PERSONAS)
+    assert template["persona_ids"] == [benchmark.DEFAULT_PERSONA_ID]
     assert template["test_set_ids"] == ["TSDENTAL"]
     assert template["metric_ids"] == ["M" * 22]
     assert template["iteration_count"] == 1
@@ -158,7 +160,7 @@ def test_sync_patches_drifted_metadata_wholesale_and_leaves_the_rest() -> None:
     state = _state(
         agents=[live],
         test_set_agents=[{"id": "A"}],
-        run_templates=[{"id": "T", "display_name": RUN_NAME}],
+        run_templates=[{**DEFINITION.run_template_body("A"), "id": "T"}],
         scheduled_runs=[{"id": "S", "run_template_id": "T"}],
     )
     with _client(state) as client:
@@ -167,12 +169,36 @@ def test_sync_patches_drifted_metadata_wholesale_and_leaves_the_rest() -> None:
     assert result.actions == [
         "agent: patch ['metadata']",
         "test set: attached",
-        "run template: exists",
+        "run template: unchanged",
         "scheduled run: exists",
     ]
     assert state["writes"] == [("/agents/A", {"metadata": DEFINITION.agent_body()["metadata"]})]
 
 
+def test_sync_patches_only_the_drifted_template_fields() -> None:
+    live_template = {
+        **DEFINITION.run_template_body("A"),
+        "id": "T",
+        "persona_ids": [benchmark.DEFAULT_PERSONA_ID, "9ATy64zKXxSUaVWb5YnQtd"],
+        "concurrency": 1,
+    }
+    state = _state(
+        agents=[{**DEFINITION.agent_body(), "id": "A"}],
+        test_set_agents=[{"id": "A"}],
+        run_templates=[live_template],
+        scheduled_runs=[{"id": "S", "run_template_id": "T"}],
+    )
+    with _client(state) as client:
+        assert (
+            sync(client, DEFINITION, dry_run=True).actions[2]
+            == "run template: patch ['persona_ids']"
+        )
+        assert state["writes"] == []
+        sync(client, DEFINITION)
+
+    assert state["writes"] == [("/run-templates/T", {"persona_id": benchmark.DEFAULT_PERSONA_ID})]
+
+
 def test_sync_looks_up_by_customer_id_filter_and_never_adopts_a_name_only_match() -> None:
     name_only = {**DEFINITION.agent_body(), "id": "B", "customer_agent_id": ""}
     state = _state(agents=[name_only])