Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion evals/trigger/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,9 @@ others about the model or the version of Claude Code.
Building the instance asks the image registries about base images, so a network that cannot
reach them stops a batch before it starts. `--reuse-images` starts from the images already
built instead, after checking that every file they copy from the repository is the same as in
your checkout, and refuses otherwise.
your checkout, and refuses otherwise. The instance's images are built as `trigger-test`, never
as a release's number, so a batch leaves the images of every release on your machine as they
were.

When a rule turns out to have been applied wrongly to recorded runs, `--reread <batch>` applies
the rule as it now stands to each run's saved stream of events. What was recorded first is kept
Expand Down
20 changes: 19 additions & 1 deletion scripts/run_trigger_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,15 @@

IMAGE = f"dpolens-trigger-run:{CLAUDE_CODE}"
PROJECT = "dpolens-trigger"
INSTANCE_VERSION = "trigger-test"
"""What the instance's images are built as. Compose names each image after DPOLENS_VERSION,
so a build under a release's number would replace that release's images on this machine."""

INSTANCE_IMAGES = {
"api": f"ghcr.io/alkhatibdev/dpolens:{INSTANCE_VERSION}",
"setup": f"ghcr.io/alkhatibdev/dpolens:{INSTANCE_VERSION}",
"mcp": f"ghcr.io/alkhatibdev/dpolens-mcp:{INSTANCE_VERSION}",
}
NETWORK = f"{PROJECT}_default"
MCP_URL = "http://mcp:8765/mcp"
DEVELOPER = "developer@larder.shop"
Expand Down Expand Up @@ -548,6 +557,7 @@ def __init__(self) -> None:
"DPOLENS_API_PORT": "18000",
"DPOLENS_MCP_PORT": "18765",
"DPOLENS_MCP_ALLOWED_HOSTS": '["mcp:8765"]',
"DPOLENS_VERSION": INSTANCE_VERSION,
}
self.token = ""

Expand Down Expand Up @@ -685,7 +695,15 @@ def stale_images() -> list[str]:
for path, name in inside.items()
}
listing = subprocess.run(
["docker", "run", "--rm", "--entrypoint", "sha256sum", f"{PROJECT}-{service}", *inside],
[
"docker",
"run",
"--rm",
"--entrypoint",
"sha256sum",
INSTANCE_IMAGES[service],
*inside,
],
capture_output=True,
text=True,
check=False,
Expand Down
41 changes: 41 additions & 0 deletions tests/unit/test_trigger_runs.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,28 @@
from __future__ import annotations

import json
import re
import subprocess
import sys
from dataclasses import asdict
from pathlib import Path
from typing import Any

import pytest
import yaml

REPO = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO / "scripts"))

import run_trigger_test # noqa: E402
from run_trigger_test import ( # noqa: E402
BAKED_IN,
INSTANCE_IMAGES,
INSTANCE_VERSION,
PREFIXES,
SERVERS,
TRIGGER,
Instance,
NotPublishable,
Observation,
Outcome,
Expand Down Expand Up @@ -307,6 +314,40 @@ def test_rereading_a_batch_rescores_what_the_old_rule_misjudged(
assert read_outcomes(batch / "outcomes.recorded.jsonl") == [misjudged]


def test_the_instance_is_built_under_a_name_no_release_has(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Compose names a built image after DPOLENS_VERSION, so a build under a release's
number would replace that release's images on the machine running the test."""
monkeypatch.setenv("DPOLENS_VERSION", "0.2.0")

assert Instance().env["DPOLENS_VERSION"] == INSTANCE_VERSION
assert not re.match(r"\d", INSTANCE_VERSION)


def test_the_images_checked_are_the_images_compose_builds(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Checking other images than the ones started would vouch for the wrong files."""
services = yaml.safe_load((REPO / "compose.yaml").read_text(encoding="utf-8"))["services"]
checked: list[str] = []

def answer(command: list[str], **_: Any) -> subprocess.CompletedProcess[str]:
if command[0] == "docker":
checked.extend(command)
return subprocess.CompletedProcess(command, 0, stdout="", stderr="")

monkeypatch.setattr(subprocess, "run", answer)
run_trigger_test.stale_images()

for service in BAKED_IN:
built = re.sub(
r"\$\{DPOLENS_VERSION:-[^}]*\}", INSTANCE_VERSION, services[service]["image"]
)
assert INSTANCE_IMAGES[service] == built
assert built in checked


def test_an_image_holding_the_checkout_differs_in_nothing() -> None:
listing = "aaa /opt/dpolens/uv.lock\nbbb /opt/dpolens/packages/x.py\n"
expected = {"/opt/dpolens/uv.lock": "aaa", "/opt/dpolens/packages/x.py": "bbb"}
Expand Down
Loading