Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion evals/trigger/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,9 @@ reach them stops a batch before it starts. `--reuse-images` starts from the imag
built instead, after checking that every file they copy from the repository is the same as in
your checkout, and refuses otherwise. The instance's images are built as `trigger-test`, never
as a release's number, so a batch leaves the images of every release on your machine as they
were.
were. The instance chooses GDPR as the law that applies when it starts, since an instance
searches nothing until its laws are chosen. A search that comes back as an error is named in the
report, and `--write` refuses a batch that has one.

When a rule turns out to have been applied wrongly to recorded runs, `--reread <batch>` applies
the rule as it now stands to each run's saved stream of events. What was recorded first is kept
Expand Down
21 changes: 21 additions & 0 deletions scripts/run_trigger_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@
"setup": f"ghcr.io/alkhatibdev/dpolens:{INSTANCE_VERSION}",
"mcp": f"ghcr.io/alkhatibdev/dpolens-mcp:{INSTANCE_VERSION}",
}
LAWS = ("gdpr",)
"""What the instance's searches cover, as in the published runs. An instance searches nothing
until somebody chooses its laws, and refuses every search until then."""

NETWORK = f"{PROJECT}_default"
MCP_URL = "http://mcp:8765/mcp"
DEVELOPER = "developer@larder.shop"
Expand Down Expand Up @@ -222,6 +226,8 @@ class Observation:
"""Where in `tools` the first counted call is."""
first_edit: int | None = None
call_answered: bool = False
call_failed: bool = False
"""Whether the first counted call came back as an error, which no setup should cause."""
dpolens_answered: int = 0
"""DPOLens calls that came back, each of which wrote one row to the query log."""
retries: list[str] = field(default_factory=list)
Expand Down Expand Up @@ -284,6 +290,7 @@ def _user(self, event: dict[str, Any]) -> None:
use = str(block.get("tool_use_id"))
if use == self._call_id:
self.call_answered = True
self.call_failed = bool(block.get("is_error"))
if self._names.get(use, "").startswith(self.prefix):
self.dpolens_answered += 1

Expand Down Expand Up @@ -349,6 +356,9 @@ class Outcome:
tools: list[str]
dpolens_calls: int
query_log_rows: int | None
search_failed: bool = False
"""Whether the search that decided the run came back as an error. Batches recorded before
this was noted read as False."""


def outcome_of(
Expand Down Expand Up @@ -385,6 +395,7 @@ def outcome_of(
tools=seen.tools,
dpolens_calls=seen.dpolens_answered,
query_log_rows=rows,
search_failed=seen.call_failed,
)


Expand Down Expand Up @@ -473,6 +484,9 @@ def runs_of(task: Task) -> list[Outcome]:
"query_log_mismatches": [
f"{outcome.task}#{outcome.run}" for outcome in scored if not agrees(outcome)
],
"searches_that_failed": [
f"{outcome.task}#{outcome.run}" for outcome in scored if outcome.search_failed
],
"estimated_cost_usd": round(sum(costs), 2),
"never_searched": [
task.id
Expand Down Expand Up @@ -525,6 +539,7 @@ def print_score(found: dict[str, Any]) -> None:
("never_searched", "never searched"),
("searched_without_need", "searched without needing to"),
("query_log_mismatches", "query log disagrees with the stream for"),
("searches_that_failed", "a search came back as an error in"),
):
if found[key]:
print(f" {label}: {', '.join(found[key])}")
Expand Down Expand Up @@ -618,6 +633,7 @@ def start(self, build: bool = True) -> None:
capture_output=True,
)
self.token = token_from(created.stdout)
self.compose("exec", "-T", "api", "dpolens", "laws", "select", *LAWS, capture_output=True)

def _retry(self, *args: str) -> None:
"""A Compose command that a network blip can fail, tried a few times."""
Expand Down Expand Up @@ -1113,6 +1129,11 @@ def published(batch_ids: Sequence[str]) -> dict[str, Any]:
expected = len(tasks) * meta["runs_per_task"]
if found["runs_scored"] != expected:
raise NotPublishable(f"{batch_id} scored {found['runs_scored']} of {expected} runs")
if found["searches_that_failed"]:
raise NotPublishable(
f"{batch_id}: a search came back as an error in "
f"{len(found['searches_that_failed'])} runs, so its instance could not answer"
)
if found["models"] != [meta["model"]] or found["versions"] != [meta["claude_code"]]:
raise NotPublishable(
f"{batch_id} pinned {meta['model']} on {meta['claude_code']}, and its sessions "
Expand Down
54 changes: 53 additions & 1 deletion tests/unit/test_trigger_runs.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
BAKED_IN,
INSTANCE_IMAGES,
INSTANCE_VERSION,
LAWS,
PREFIXES,
SERVERS,
TRIGGER,
Expand Down Expand Up @@ -75,6 +76,23 @@ def answer(tool_id: str) -> dict[str, Any]:
}


def refused(tool_id: str) -> dict[str, Any]:
return {
"type": "user",
"message": {
"content": [
{
"type": "tool_result",
"tool_use_id": tool_id,
"is_error": True,
"content": "Error executing tool search_policies: nobody has chosen which "
"laws apply to this organisation yet, so nothing is searched.",
}
]
},
}


def watch(*events: dict[str, Any]) -> Observation:
seen = Observation(prefix=PREFIXES["plugin"], server=SERVERS["plugin"])
for event in events:
Expand All @@ -96,6 +114,37 @@ def test_the_token_is_the_line_that_holds_it_not_the_prefix_named_above_it() ->
assert token_from(printed) == "dpol_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789_-abc"


def test_a_search_that_came_back_as_an_error_is_noted() -> None:
"""The decision to search still counts; the report says the instance could not answer."""
seen = watch(init(), use("1", SEARCH), refused("1"))
task = Task(id="t", kind="coding", should_search=True, prompt="p", reason="r", verb="store")
outcome = outcome_of(
task, 1, 1, "plugin", seen, stopped_at_call=True, clock=False, seconds=1.0, rows=0
)

assert seen.called and seen.call_failed
assert outcome.scored and outcome.search_failed
assert score([task], [outcome])["searches_that_failed"] == ["t#1"]


def test_the_instance_chooses_the_law_its_searches_cover(monkeypatch: pytest.MonkeyPatch) -> None:
"""An instance searches nothing until its laws are chosen, and refuses every search."""
commands: list[tuple[str, ...]] = []

def compose(_: Instance, *args: str, **__: Any) -> subprocess.CompletedProcess[str]:
commands.append(args)
printed = "dpol_" + "a" * 40 if "token" in args else ""
return subprocess.CompletedProcess(args, 0, stdout=printed, stderr="")

monkeypatch.setattr(Instance, "compose", compose)
monkeypatch.setattr(Instance, "_retry", lambda *_: None)
monkeypatch.setattr(Instance, "_wait_until_healthy", lambda *_, **__: None)

Instance().start()

assert ("exec", "-T", "api", "dpolens", "laws", "select", *LAWS) in commands


def test_a_search_before_any_edit_is_a_call_before_editing() -> None:
seen = watch(init(), use("1", "Read"), use("2", SEARCH), answer("2"), use("3", "Edit"))

Expand Down Expand Up @@ -377,6 +426,7 @@ def held_out_batch(
question_set: str = "held_out",
model: str | None = "claude-sonnet-5-5",
drop: int = 0,
failed: bool = False,
) -> None:
"""A batch in which every task did what its label says, once."""
batch = runs / name
Expand All @@ -393,7 +443,8 @@ def held_out_batch(
session = {**init(), "model": model or "claude-sonnet-5-5"}
lines = []
for task in read_tasks(TRIGGER / "held_out.yaml"):
events = [session, *([use("1", SEARCH), answer("1")] if task.should_search else [])]
reply = refused("1") if failed else answer("1")
events = [session, *([use("1", SEARCH), reply] if task.should_search else [])]
seen = watch(*events)
result = outcome_of(
task,
Expand Down Expand Up @@ -432,6 +483,7 @@ def test_a_complete_pinned_held_out_batch_is_published(
({"question_set": "tuning"}, "only held-out is published"),
({"model": None}, "not a pinned one"),
({"drop": 1}, "scored 29 of 30 runs"),
({"failed": True}, "came back as an error in 15 runs"),
],
)
def test_a_batch_that_cannot_stand_behind_the_number_is_refused(
Expand Down
Loading