diff --git a/evals/trigger/README.md b/evals/trigger/README.md index 03bad11..8f0b8bc 100644 --- a/evals/trigger/README.md +++ b/evals/trigger/README.md @@ -102,7 +102,9 @@ reach them stops a batch before it starts. `--reuse-images` starts from the imag built instead, after checking that every file they copy from the repository is the same as in your checkout, and refuses otherwise. The instance's images are built as `trigger-test`, never as a release's number, so a batch leaves the images of every release on your machine as they -were. +were. The instance chooses GDPR as the law that applies when it starts, since an instance +searches nothing until its laws are chosen. A search that comes back as an error is named in the +report, and `--write` refuses a batch that has one. When a rule turns out to have been applied wrongly to recorded runs, `--reread ` applies the rule as it now stands to each run's saved stream of events. What was recorded first is kept diff --git a/scripts/run_trigger_test.py b/scripts/run_trigger_test.py index 5f2cfff..4874d35 100644 --- a/scripts/run_trigger_test.py +++ b/scripts/run_trigger_test.py @@ -64,6 +64,10 @@ "setup": f"ghcr.io/alkhatibdev/dpolens:{INSTANCE_VERSION}", "mcp": f"ghcr.io/alkhatibdev/dpolens-mcp:{INSTANCE_VERSION}", } +LAWS = ("gdpr",) +"""What the instance's searches cover, as in the published runs. An instance searches nothing +until somebody chooses its laws, and refuses every search until then.""" + NETWORK = f"{PROJECT}_default" MCP_URL = "http://mcp:8765/mcp" DEVELOPER = "developer@larder.shop" @@ -222,6 +226,8 @@ class Observation: """Where in `tools` the first counted call is.""" first_edit: int | None = None call_answered: bool = False + call_failed: bool = False + """Whether the first counted call came back as an error, which no setup should cause.""" dpolens_answered: int = 0 """DPOLens calls that came back, each of which wrote one row to the query log.""" retries: list[str] = field(default_factory=list) @@ -284,6 +290,7 @@ def _user(self, event: dict[str, Any]) -> None: use = str(block.get("tool_use_id")) if use == self._call_id: self.call_answered = True + self.call_failed = bool(block.get("is_error")) if self._names.get(use, "").startswith(self.prefix): self.dpolens_answered += 1 @@ -349,6 +356,9 @@ class Outcome: tools: list[str] dpolens_calls: int query_log_rows: int | None + search_failed: bool = False + """Whether the search that decided the run came back as an error. Batches recorded before + this was noted read as False.""" def outcome_of( @@ -385,6 +395,7 @@ def outcome_of( tools=seen.tools, dpolens_calls=seen.dpolens_answered, query_log_rows=rows, + search_failed=seen.call_failed, ) @@ -473,6 +484,9 @@ def runs_of(task: Task) -> list[Outcome]: "query_log_mismatches": [ f"{outcome.task}#{outcome.run}" for outcome in scored if not agrees(outcome) ], + "searches_that_failed": [ + f"{outcome.task}#{outcome.run}" for outcome in scored if outcome.search_failed + ], "estimated_cost_usd": round(sum(costs), 2), "never_searched": [ task.id @@ -525,6 +539,7 @@ def print_score(found: dict[str, Any]) -> None: ("never_searched", "never searched"), ("searched_without_need", "searched without needing to"), ("query_log_mismatches", "query log disagrees with the stream for"), + ("searches_that_failed", "a search came back as an error in"), ): if found[key]: print(f" {label}: {', '.join(found[key])}") @@ -618,6 +633,7 @@ def start(self, build: bool = True) -> None: capture_output=True, ) self.token = token_from(created.stdout) + self.compose("exec", "-T", "api", "dpolens", "laws", "select", *LAWS, capture_output=True) def _retry(self, *args: str) -> None: """A Compose command that a network blip can fail, tried a few times.""" @@ -1113,6 +1129,11 @@ def published(batch_ids: Sequence[str]) -> dict[str, Any]: expected = len(tasks) * meta["runs_per_task"] if found["runs_scored"] != expected: raise NotPublishable(f"{batch_id} scored {found['runs_scored']} of {expected} runs") + if found["searches_that_failed"]: + raise NotPublishable( + f"{batch_id}: a search came back as an error in " + f"{len(found['searches_that_failed'])} runs, so its instance could not answer" + ) if found["models"] != [meta["model"]] or found["versions"] != [meta["claude_code"]]: raise NotPublishable( f"{batch_id} pinned {meta['model']} on {meta['claude_code']}, and its sessions " diff --git a/tests/unit/test_trigger_runs.py b/tests/unit/test_trigger_runs.py index 83a469b..521cefe 100644 --- a/tests/unit/test_trigger_runs.py +++ b/tests/unit/test_trigger_runs.py @@ -26,6 +26,7 @@ BAKED_IN, INSTANCE_IMAGES, INSTANCE_VERSION, + LAWS, PREFIXES, SERVERS, TRIGGER, @@ -75,6 +76,23 @@ def answer(tool_id: str) -> dict[str, Any]: } +def refused(tool_id: str) -> dict[str, Any]: + return { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": tool_id, + "is_error": True, + "content": "Error executing tool search_policies: nobody has chosen which " + "laws apply to this organisation yet, so nothing is searched.", + } + ] + }, + } + + def watch(*events: dict[str, Any]) -> Observation: seen = Observation(prefix=PREFIXES["plugin"], server=SERVERS["plugin"]) for event in events: @@ -96,6 +114,37 @@ def test_the_token_is_the_line_that_holds_it_not_the_prefix_named_above_it() -> assert token_from(printed) == "dpol_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789_-abc" +def test_a_search_that_came_back_as_an_error_is_noted() -> None: + """The decision to search still counts; the report says the instance could not answer.""" + seen = watch(init(), use("1", SEARCH), refused("1")) + task = Task(id="t", kind="coding", should_search=True, prompt="p", reason="r", verb="store") + outcome = outcome_of( + task, 1, 1, "plugin", seen, stopped_at_call=True, clock=False, seconds=1.0, rows=0 + ) + + assert seen.called and seen.call_failed + assert outcome.scored and outcome.search_failed + assert score([task], [outcome])["searches_that_failed"] == ["t#1"] + + +def test_the_instance_chooses_the_law_its_searches_cover(monkeypatch: pytest.MonkeyPatch) -> None: + """An instance searches nothing until its laws are chosen, and refuses every search.""" + commands: list[tuple[str, ...]] = [] + + def compose(_: Instance, *args: str, **__: Any) -> subprocess.CompletedProcess[str]: + commands.append(args) + printed = "dpol_" + "a" * 40 if "token" in args else "" + return subprocess.CompletedProcess(args, 0, stdout=printed, stderr="") + + monkeypatch.setattr(Instance, "compose", compose) + monkeypatch.setattr(Instance, "_retry", lambda *_: None) + monkeypatch.setattr(Instance, "_wait_until_healthy", lambda *_, **__: None) + + Instance().start() + + assert ("exec", "-T", "api", "dpolens", "laws", "select", *LAWS) in commands + + def test_a_search_before_any_edit_is_a_call_before_editing() -> None: seen = watch(init(), use("1", "Read"), use("2", SEARCH), answer("2"), use("3", "Edit")) @@ -377,6 +426,7 @@ def held_out_batch( question_set: str = "held_out", model: str | None = "claude-sonnet-5-5", drop: int = 0, + failed: bool = False, ) -> None: """A batch in which every task did what its label says, once.""" batch = runs / name @@ -393,7 +443,8 @@ def held_out_batch( session = {**init(), "model": model or "claude-sonnet-5-5"} lines = [] for task in read_tasks(TRIGGER / "held_out.yaml"): - events = [session, *([use("1", SEARCH), answer("1")] if task.should_search else [])] + reply = refused("1") if failed else answer("1") + events = [session, *([use("1", SEARCH), reply] if task.should_search else [])] seen = watch(*events) result = outcome_of( task, @@ -432,6 +483,7 @@ def test_a_complete_pinned_held_out_batch_is_published( ({"question_set": "tuning"}, "only held-out is published"), ({"model": None}, "not a pinned one"), ({"drop": 1}, "scored 29 of 30 runs"), + ({"failed": True}, "came back as an error in 15 runs"), ], ) def test_a_batch_that_cannot_stand_behind_the_number_is_refused(