From 2ecf2749ae6fb16b745c103692fe63a16fff9b3a Mon Sep 17 00:00:00 2001 From: alkhatibdev Date: Thu, 19 Nov 2026 12:05:08 +0400 Subject: [PATCH] feat: the trigger test measured again with the narrowed skill, none of 45 Signed-off-by: alkhatibdev --- README.md | 2 +- evals/results/trigger.json | 89 ++++++++++++++++++--------------- evals/trigger-test.md | 86 ++++++++++++++++--------------- evals/trigger/README.md | 4 ++ scripts/run_trigger_test.py | 31 ++++++++++-- tests/unit/test_trigger_runs.py | 64 +++++++++++++++++++++--- 6 files changed, 184 insertions(+), 92 deletions(-) diff --git a/README.md b/README.md index 5e78332..54cdf5e 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ Code, Codex, Cursor and GitHub Copilot in VS Code. answers 73% of the time for GDPR, and 53% for the UAE's law. [How it was measured](evals/retrieval.md) - **Use:** Claude Code with the plugin searched before writing code in 36 of 36 runs of tasks - that touch personal data, and in 3 of 45 runs of tasks that do not. + that touch personal data, and in none of the 45 runs of tasks that do not. [How it was measured](evals/trigger-test.md) ## Contributing diff --git a/evals/results/trigger.json b/evals/results/trigger.json index 0209620..45a690b 100644 --- a/evals/results/trigger.json +++ b/evals/results/trigger.json @@ -1,35 +1,26 @@ { "question_set": "held_out", - "measured_at": "2026-10-09", + "measured_at": "2026-11-19", "model": "claude-sonnet-5-5", "claude_code": "2.1.288", "runs": [ { "setup": "plugin", "batch": { - "id": "held_out-plugin", + "id": "held_out-plugin-2026-11", "set": "held_out", "setup": "plugin", "model": "claude-sonnet-5-5", + "client": "claude-code", + "client_version": "2.1.288", "claude_code": "2.1.288", "runs_per_task": 3, "max_turns": 25, "max_budget_usd": null, - "commit": "1260d14", - "started": "2026-10-08T19:44:46+00:00", - "resumed": [ - { - "started": "2026-10-08T20:35:58+00:00", - "commit": "1260d14", - "uncommitted_changes": true - }, - { - "started": "2026-10-09T06:51:59+00:00", - "commit": "1260d14", - "uncommitted_changes": true, - "instance": "reused images holding the same files as the checkout" - } - ] + "started": "2026-11-18T21:28:11+00:00", + "commit": "bf5b595", + "uncommitted_changes": false, + "instance": "built from the checkout" }, "groups": { "should": { @@ -56,13 +47,13 @@ "tasks": 15, "rate": { "runs": 45, - "calls": 3, + "calls": 0, "low": 0.0, - "high": 0.2 + "high": 0.0 }, "consistency": { - "every": 1, - "none": 14, + "every": 0, + "none": 15, "some": 0 } }, @@ -82,14 +73,16 @@ } }, "runs_scored": 90, - "attempts_not_scored": 2, + "attempts_not_scored": 0, "capped": 0, - "query_log_mismatches": [], - "estimated_cost_usd": 2.17, - "never_searched": [], - "searched_without_need": [ - "product-viewed-event" + "query_log_mismatches": [ + "email-in-order-event#2" ], + "searches_that_failed": [], + "searches_without_a_question": [], + "estimated_cost_usd": 2.35, + "never_searched": [], + "searched_without_need": [], "models": [ "claude-sonnet-5-5" ], @@ -100,18 +93,28 @@ { "setup": "mcp-only", "batch": { - "id": "held_out-mcp-only", + "id": "held_out-mcp-only-2026-11", "set": "held_out", "setup": "mcp-only", "model": "claude-sonnet-5-5", + "client": "claude-code", + "client_version": "2.1.288", "claude_code": "2.1.288", "runs_per_task": 3, "max_turns": 25, "max_budget_usd": null, - "started": "2026-10-09T06:54:36+00:00", - "commit": "1260d14", - "uncommitted_changes": true, - "instance": "reused images holding the same files as the checkout" + "started": "2026-11-18T21:56:09+00:00", + "commit": "bf5b595", + "uncommitted_changes": false, + "instance": "built from the checkout", + "resumed": [ + { + "started": "2026-11-19T06:48:54+00:00", + "commit": "bf5b595", + "uncommitted_changes": false, + "instance": "built from the checkout" + } + ] }, "groups": { "should": { @@ -138,14 +141,14 @@ "tasks": 15, "rate": { "runs": 45, - "calls": 0, + "calls": 2, "low": 0.0, - "high": 0.0 + "high": 0.13333333333333333 }, "consistency": { "every": 0, - "none": 15, - "some": 0 + "none": 14, + "some": 1 } }, "questions": { @@ -164,14 +167,20 @@ } }, "runs_scored": 90, - "attempts_not_scored": 0, + "attempts_not_scored": 2, "capped": 0, "query_log_mismatches": [ - "copy-request-question#1" + "ip-logging-question#1" + ], + "searches_that_failed": [], + "searches_without_a_question": [ + "ip-logging-question#1" ], - "estimated_cost_usd": 2.33, + "estimated_cost_usd": 2.21, "never_searched": [], - "searched_without_need": [], + "searched_without_need": [ + "product-viewed-event" + ], "models": [ "claude-sonnet-5-5" ], diff --git a/evals/trigger-test.md b/evals/trigger-test.md index b6c63ef..108bb00 100644 --- a/evals/trigger-test.md +++ b/evals/trigger-test.md @@ -8,14 +8,14 @@ rest depends on. ## The published result **Claude Code with the DPOLens plugin searched before writing code in 36 of 36 runs of tasks -that touch personal data, and in 3 of 45 runs of tasks that do not.** Asked questions about the -rules, it searched in 9 of 9. Thirty held-out tasks, three runs each, Claude Code 2.1.288 with -`claude-sonnet-5-5`, measured 8 and 9 October 2026. +that touch personal data, and in none of the 45 runs of tasks that do not.** Asked questions +about the rules, it searched in 9 of 9. Thirty held-out tasks, three runs each, Claude Code +2.1.288 with `claude-sonnet-5-5`, measured 18 and 19 November 2026. | Group | Tasks | Searched | Share | 95% interval | | --- | --- | --- | --- | --- | | Coding tasks that touch personal data | 12 | 36 of 36 runs, all before the first edit | 1.00 | 1.00 to 1.00 | -| Coding tasks that do not | 15 | 3 of 45 runs | 0.07 | 0.00 to 0.20 | +| Coding tasks that do not | 15 | 0 of 45 runs | 0.00 | 0.00 to 0.00 | | Questions about the rules | 3 | 9 of 9 runs | 1.00 | 1.00 to 1.00 | Every task behaved the same way in all three of its runs: no task searched in some runs and not @@ -24,20 +24,34 @@ others. Read the first line strictly. A bootstrap interval cannot show spread when every task succeeds every time, so "1.00 to 1.00" is not certainty. With twelve tasks and no misses, the honest bound is the rule of three: the share of tasks like these that would miss is likely below one in -four. It says that on these tasks, in this app, Claude Code with the plugin searched every time -it should have, and before it touched the code. It does not say that every AI assistant, or every -codebase, behaves the same way. +four. The same goes for none of 45: with fifteen tasks and no search they did not need, the +share of tasks like these that would search without need is likely below one in five. It says +that on these tasks, in this app, Claude Code with the plugin searched every time it should +have, before it touched the code, and never when it did not need to. It does not say that every +AI assistant, or every codebase, behaves the same way. `evals/results/trigger.json` carries the same numbers, with the record of how each batch ran. -## The task that searched without needing to +## What changed since October -All three searches that should not have happened are one task, `product-viewed-event`, in each of -its runs. It asks for a `product_viewed` event to be sent to Segment when a product page opens, -with the product slug. The label says no search is needed, because the event carries nothing -about the person viewing the page. The AI assistant asked about "sending product page view -analytics events to a third-party service such as Segment", which is a fair question, and the -number counts it against DPOLens anyway, because that is what the label says. +The first published result, measured 8 and 9 October 2026, had the plugin searching in 3 of 45 +runs of tasks that do not touch personal data. All three were one task, `product-viewed-event`, +which asks for a `product_viewed` event to be sent to Segment when a product page opens, with the +product slug. The event carries nothing about the person viewing the page, so the label says no +search is needed, and the number counts the search against DPOLens. + +The task had two ways in. The skill's description said to use DPOLens "when sending data to +another service", which that task does with data about no one, where the server's instructions +say "personal data". And in one of its three runs, Claude Code reached DPOLens through the +plugin's `/policy-check` command, which it chose to run on its own. The skill now says "when +sending personal data to another service", and the result above was measured with it: +`product-viewed-event` searched in none of its runs, and `/policy-check` never ran on its own. A +first rerun of the same change, on 17 and 18 November with an instance that refused every search +because no law was chosen, had one search it did not need in 45: `session-length` ("Make sessions +last 7 days instead of 30"), in one of its three runs. So read none of 45 as rarely, not never. + +One task was reworded between the two measurements, `id-check-on-wine` to +`id-check-on-delivery`, so both setups were measured again on the tasks as they stand. ## Without the plugin @@ -48,25 +62,16 @@ added on its own, so that only the instructions the server sends when it connect | Group | With the plugin | MCP server on its own | | --- | --- | --- | | Coding tasks that touch personal data | 36 of 36 | 36 of 36 | -| Coding tasks that do not | 3 of 45 | 0 of 45 | +| Coding tasks that do not | 0 of 45 | 2 of 45 | | Questions about the rules | 9 of 9 | 9 of 9 | -The server's instructions did the work: without the skill, every task that should search still -did, before editing. The one difference is `product-viewed-event`, and it had two ways in. The -skill's description said to use DPOLens "when sending data to another service", which that task -does with data about no one, where the server's instructions say "personal data". And in one of -its three runs, Claude Code reached DPOLens through the plugin's `/policy-check` command, which -it chose to run on its own. Without the plugin, the task never searched. - -The skill's description now says "when sending personal data to another service", and the plugin -arm was run again with that one change, on 17 and 18 November 2026. It searched in 36 of 36 runs -of tasks that touch personal data, all before the first edit, in 1 of 45 runs of tasks that do -not, and in 9 of 9 runs of questions about the rules. `product-viewed-event` searched in none of -its three runs, and `/policy-check` never ran on its own. The one search that should not have -happened was `session-length` ("Make sessions last 7 days instead of 30"), in one of its three -runs, asking "how long can session records and login tokens be retained". The rerun's instance -had no law chosen, so every search came back as an error. That does not change the count: a run -is scored on the call, and the call comes before the answer. +The server's instructions do the work: without the skill, every task that should search still +did, before editing. This time the one difference runs the other way. Without the plugin, +`product-viewed-event` searched in 2 of its 3 runs, asking whether page-view events "tied to a +user or anonymous visitor identifier" can go to Segment. That is a fair question, since a visitor +identifier can be personal data, and the label counts it against DPOLens all the same. In +October, without the plugin, the task never searched, and the instructions the server sends are +the same text as then. The plugin is still what [Get started](https://dpolens.com/docs/get-started) installs, for its commands and for keeping the token in the system's credential store. @@ -98,15 +103,16 @@ that return clause text. It searched before editing if that came before its firs `Write`, `MultiEdit` or `NotebookEdit`. A run stops once its first search comes back, since the outcome is known by then. Otherwise it runs until the AI assistant finishes, 25 turns or ten minutes. Every run's searches are also counted in the instance's query log, and the two counts -agreed for every run but one. Without the plugin, one run called `search_policies` with no +agreed for every run but two. Without the plugin, one run called `search_policies` with no question at all, before loading the tool's description, and the server refused the call, so it never reached the log. It counts as a search, because what is measured is whether the AI assistant -decides to look the rules up, and the run stopped there. +decides to look the rules up, and the run stopped there. With the plugin, one run's log holds a +second question that the run's record, cut at the first answer, never shows. -**Runs that could not be scored.** Two runs of one task, `dark-theme`, spent their ten minutes -retrying requests to the model that the network never answered. A run that could not have -reached DPOLens is not a run that chose not to, so both were run again and are reported as -attempts not scored. +**Runs that could not be scored.** Without the plugin, two attempts at one run of +`welcome-email-template` failed when the subscription's usage limit was reached, and the run was +made again after the limit reset. A run that could not have reached DPOLens is not a run that +chose not to, so both are reported as attempts not scored. **The interval.** Tasks are resampled with their three runs kept together, because three runs of one task are not three tasks. @@ -130,9 +136,9 @@ and reading the full record each run leaves behind. ## What it does not show -- **Other AI assistants.** Only Claude Code was measured, on one model, on one date. Codex, Cursor - and GitHub Copilot in VS Code read the same guidance, but how often they act on it has not been - measured. +- **Other AI assistants.** Only Claude Code was measured, on one model and one version. Codex, + Cursor and GitHub Copilot in VS Code read the same guidance, but how often they act on it has + not been measured. - **Messier work.** The tasks are short, plainly phrased requests in a small app. In a large codebase, the code that handles people's data is harder to spot, and requests are rarely this clear. diff --git a/evals/trigger/README.md b/evals/trigger/README.md index c725c2b..9fe64c1 100644 --- a/evals/trigger/README.md +++ b/evals/trigger/README.md @@ -72,6 +72,10 @@ known: runs stay short on a slow model, and the batch cannot say what a run woul A run that could not have called DPOLens is retried rather than scored: the plugin not loading, the MCP server not connecting, or the request to the model failing. Each run's calls are also counted in the instance's query log, and the report names any run where the two counts differ. +It also names a run whose deciding search came back as an error, which means the instance could +not answer and keeps the batch from being published, and apart from it, a run whose search asked +no question at all. The server refuses that one whatever the instance holds, and it still counts +as the decision to search. ## Running it diff --git a/scripts/run_trigger_test.py b/scripts/run_trigger_test.py index d3489a3..ee4326e 100644 --- a/scripts/run_trigger_test.py +++ b/scripts/run_trigger_test.py @@ -137,6 +137,9 @@ def version_arg(self) -> str: COUNTED = ("search_policies", "get_clause") """The tools that return clause text. Listing documents is not consulting them.""" +QUESTIONS = {"search_policies": "query", "get_clause": "key"} +"""What each counted tool needs to be asked. A call without it is refused by the server's +schema, whatever the instance holds.""" VERBS = ("collect", "store", "log", "share", "delete") KINDS = ("coding", "question") @@ -287,6 +290,9 @@ class Observation: call_answered: bool = False call_failed: bool = False """Whether the first counted call came back as an error, which no setup should cause.""" + call_asked: bool = True + """Whether the first counted call carried its question. One sent with none is the AI + assistant's slip: no instance can answer it, and the decision to search was still made.""" dpolens_answered: int = 0 """DPOLens calls that came back, each of which wrote one row to the query log.""" retries: list[str] = field(default_factory=list) @@ -352,6 +358,7 @@ def _assistant(self, event: dict[str, Any]) -> None: if self.first_call is None and self.counts(name): self.first_call = len(self.tools) - 1 self._call_id = str(block.get("id")) + self.call_asked = self._asked(block, name) if self.first_edit is None and name in self.client.edits: self.first_edit = len(self.tools) - 1 @@ -368,6 +375,15 @@ def _target(self, block: dict[str, Any]) -> str: same = {tool for tool in self._registered if tool.lower() == named.lower()} return same.pop() if len(same) == 1 else named + def _asked(self, block: dict[str, Any], name: str) -> bool: + """Whether a counted call named its question. A call through the bridge carries its + arguments in its own shape, so it is taken as asked.""" + given = block.get("input") + if block.get("name") != name or not isinstance(given, dict): + return True + question = given.get(QUESTIONS[name.removeprefix(self.prefix)]) + return isinstance(question, str) and bool(question.strip()) + def _user(self, event: dict[str, Any]) -> None: for block in blocks(event): if block.get("type") != "tool_result": @@ -460,8 +476,12 @@ class Outcome: dpolens_calls: int query_log_rows: int | None search_failed: bool = False - """Whether the search that decided the run came back as an error. Batches recorded before - this was noted read as False.""" + """Whether the search that decided the run came back as an error although it asked + something, so the instance could not answer. Batches recorded before this was noted read as + False.""" + search_asked_nothing: bool = False + """Whether the search that decided the run carried no question, which the server refuses. + It counts as a search: what is measured is the decision to look the rules up.""" stopped_at_edit: bool = False """Whether the run was ended at its first edit, which only `--stop-at-edit` does.""" @@ -501,7 +521,8 @@ def outcome_of( tools=seen.tools, dpolens_calls=seen.dpolens_answered, query_log_rows=rows, - search_failed=seen.call_failed, + search_failed=seen.call_failed and seen.call_asked, + search_asked_nothing=seen.called and not seen.call_asked, stopped_at_edit=stopped_at_edit, ) @@ -594,6 +615,9 @@ def runs_of(task: Task) -> list[Outcome]: "searches_that_failed": [ f"{outcome.task}#{outcome.run}" for outcome in scored if outcome.search_failed ], + "searches_without_a_question": [ + f"{outcome.task}#{outcome.run}" for outcome in scored if outcome.search_asked_nothing + ], "estimated_cost_usd": round(sum(costs), 2), "never_searched": [ task.id @@ -648,6 +672,7 @@ def print_score(found: dict[str, Any], client: Client) -> None: ("searched_without_need", "searched without needing to"), ("query_log_mismatches", "query log disagrees with the stream for"), ("searches_that_failed", "a search came back as an error in"), + ("searches_without_a_question", "a search asked no question in"), ): if found[key]: print(f" {label}: {', '.join(found[key])}") diff --git a/tests/unit/test_trigger_runs.py b/tests/unit/test_trigger_runs.py index db52a95..b8fc68f 100644 --- a/tests/unit/test_trigger_runs.py +++ b/tests/unit/test_trigger_runs.py @@ -71,11 +71,12 @@ def init(status: str = "connected", plugins: tuple[str, ...] = ("dpolens",)) -> } -def use(tool_id: str, name: str) -> dict[str, Any]: - return { - "type": "assistant", - "message": {"content": [{"type": "tool_use", "id": tool_id, "name": name, "input": {}}]}, - } +def use(tool_id: str, name: str, given: dict[str, Any] | None = None) -> dict[str, Any]: + block = {"type": "tool_use", "id": tool_id, "name": name, "input": given or {}} + return {"type": "assistant", "message": {"content": [block]}} + + +ASKED = {"query": "how long are failed logins kept"} def answer(tool_id: str) -> dict[str, Any]: @@ -125,7 +126,7 @@ def test_the_token_is_the_line_that_holds_it_not_the_prefix_named_above_it() -> def test_a_search_that_came_back_as_an_error_is_noted() -> None: """The decision to search still counts; the report says the instance could not answer.""" - seen = watch(init(), use("1", SEARCH), refused("1")) + seen = watch(init(), use("1", SEARCH, ASKED), refused("1")) task = Task(id="t", kind="coding", should_search=True, prompt="p", reason="r", verb="store") outcome = outcome_of( task, 1, 1, "plugin", seen, stopped_at_call=True, clock=False, seconds=1.0, rows=0 @@ -136,6 +137,37 @@ def test_a_search_that_came_back_as_an_error_is_noted() -> None: assert score([task], [outcome])["searches_that_failed"] == ["t#1"] +def test_a_search_that_asked_nothing_counts_and_is_not_the_instance_failing() -> None: + """Claude Code once called search_policies with no question before loading its description, + and the server's schema refused it. Any instance would have: the decision still counts.""" + schema = { + "type": "user", + "message": { + "content": [ + { + "type": "tool_result", + "tool_use_id": "1", + "is_error": True, + "content": "Error executing tool search_policies: 1 validation error for " + "search_policiesArguments\nquery\n Field required", + } + ] + }, + } + seen = watch(init(), use("1", SEARCH), schema) + task = Task(id="t", kind="coding", should_search=True, prompt="p", reason="r", verb="store") + outcome = outcome_of( + task, 1, 1, "plugin", seen, stopped_at_call=True, clock=False, seconds=1.0, rows=0 + ) + + assert seen.called and seen.call_failed and not seen.call_asked + assert outcome.scored and outcome.before_edit + assert not outcome.search_failed and outcome.search_asked_nothing + found = score([task], [outcome]) + assert found["searches_that_failed"] == [] + assert found["searches_without_a_question"] == ["t#1"] + + def test_the_instance_chooses_the_law_its_searches_cover(monkeypatch: pytest.MonkeyPatch) -> None: """An instance searches nothing until its laws are chosen, and refuses every search.""" commands: list[tuple[str, ...]] = [] @@ -436,6 +468,7 @@ def held_out_batch( model: str | None = "claude-sonnet-5-5", drop: int = 0, failed: bool = False, + asked_nothing: bool = False, conditions: dict[str, Any] | None = None, ) -> None: """A batch in which every task did what its label says, once.""" @@ -454,8 +487,9 @@ def held_out_batch( session = {**init(), "model": model or "claude-sonnet-5-5"} lines = [] for task in read_tasks(TRIGGER / "held_out.yaml"): - reply = refused("1") if failed else answer("1") - events = [session, *([use("1", SEARCH), reply] if task.should_search else [])] + reply = refused("1") if failed or asked_nothing else answer("1") + call = use("1", SEARCH, None if asked_nothing else ASKED) + events = [session, *([call, reply] if task.should_search else [])] seen = watch(*events) result = outcome_of( task, @@ -509,6 +543,20 @@ def test_a_batch_that_cannot_stand_behind_the_number_is_refused( published(["batch"]) +def test_a_batch_whose_searches_asked_nothing_is_published( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A search sent with no question is refused whatever the instance holds, so it says nothing + against the instance, and it still counts as the decision to search.""" + monkeypatch.setattr(run_trigger_test, "RUNS", tmp_path) + held_out_batch(tmp_path, "batch", asked_nothing=True) + + [run] = published(["batch"])["runs"] + + assert run["groups"]["should"]["rate"]["calls"] == 12 + assert len(run["searches_without_a_question"]) == 15 + + def test_batches_on_different_models_are_not_published_together( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: