diff --git a/evals/trigger/README.md b/evals/trigger/README.md index 3d7781f..03bad11 100644 --- a/evals/trigger/README.md +++ b/evals/trigger/README.md @@ -100,7 +100,9 @@ others about the model or the version of Claude Code. Building the instance asks the image registries about base images, so a network that cannot reach them stops a batch before it starts. `--reuse-images` starts from the images already built instead, after checking that every file they copy from the repository is the same as in -your checkout, and refuses otherwise. +your checkout, and refuses otherwise. The instance's images are built as `trigger-test`, never +as a release's number, so a batch leaves the images of every release on your machine as they +were. When a rule turns out to have been applied wrongly to recorded runs, `--reread ` applies the rule as it now stands to each run's saved stream of events. What was recorded first is kept diff --git a/scripts/run_trigger_test.py b/scripts/run_trigger_test.py index 0f6e646..5f2cfff 100644 --- a/scripts/run_trigger_test.py +++ b/scripts/run_trigger_test.py @@ -55,6 +55,15 @@ IMAGE = f"dpolens-trigger-run:{CLAUDE_CODE}" PROJECT = "dpolens-trigger" +INSTANCE_VERSION = "trigger-test" +"""What the instance's images are built as. Compose names each image after DPOLENS_VERSION, +so a build under a release's number would replace that release's images on this machine.""" + +INSTANCE_IMAGES = { + "api": f"ghcr.io/alkhatibdev/dpolens:{INSTANCE_VERSION}", + "setup": f"ghcr.io/alkhatibdev/dpolens:{INSTANCE_VERSION}", + "mcp": f"ghcr.io/alkhatibdev/dpolens-mcp:{INSTANCE_VERSION}", +} NETWORK = f"{PROJECT}_default" MCP_URL = "http://mcp:8765/mcp" DEVELOPER = "developer@larder.shop" @@ -548,6 +557,7 @@ def __init__(self) -> None: "DPOLENS_API_PORT": "18000", "DPOLENS_MCP_PORT": "18765", "DPOLENS_MCP_ALLOWED_HOSTS": '["mcp:8765"]', + "DPOLENS_VERSION": INSTANCE_VERSION, } self.token = "" @@ -685,7 +695,15 @@ def stale_images() -> list[str]: for path, name in inside.items() } listing = subprocess.run( - ["docker", "run", "--rm", "--entrypoint", "sha256sum", f"{PROJECT}-{service}", *inside], + [ + "docker", + "run", + "--rm", + "--entrypoint", + "sha256sum", + INSTANCE_IMAGES[service], + *inside, + ], capture_output=True, text=True, check=False, diff --git a/tests/unit/test_trigger_runs.py b/tests/unit/test_trigger_runs.py index b8b12e1..83a469b 100644 --- a/tests/unit/test_trigger_runs.py +++ b/tests/unit/test_trigger_runs.py @@ -8,21 +8,28 @@ from __future__ import annotations import json +import re +import subprocess import sys from dataclasses import asdict from pathlib import Path from typing import Any import pytest +import yaml REPO = Path(__file__).resolve().parents[2] sys.path.insert(0, str(REPO / "scripts")) import run_trigger_test # noqa: E402 from run_trigger_test import ( # noqa: E402 + BAKED_IN, + INSTANCE_IMAGES, + INSTANCE_VERSION, PREFIXES, SERVERS, TRIGGER, + Instance, NotPublishable, Observation, Outcome, @@ -307,6 +314,40 @@ def test_rereading_a_batch_rescores_what_the_old_rule_misjudged( assert read_outcomes(batch / "outcomes.recorded.jsonl") == [misjudged] +def test_the_instance_is_built_under_a_name_no_release_has( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Compose names a built image after DPOLENS_VERSION, so a build under a release's + number would replace that release's images on the machine running the test.""" + monkeypatch.setenv("DPOLENS_VERSION", "0.2.0") + + assert Instance().env["DPOLENS_VERSION"] == INSTANCE_VERSION + assert not re.match(r"\d", INSTANCE_VERSION) + + +def test_the_images_checked_are_the_images_compose_builds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Checking other images than the ones started would vouch for the wrong files.""" + services = yaml.safe_load((REPO / "compose.yaml").read_text(encoding="utf-8"))["services"] + checked: list[str] = [] + + def answer(command: list[str], **_: Any) -> subprocess.CompletedProcess[str]: + if command[0] == "docker": + checked.extend(command) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + monkeypatch.setattr(subprocess, "run", answer) + run_trigger_test.stale_images() + + for service in BAKED_IN: + built = re.sub( + r"\$\{DPOLENS_VERSION:-[^}]*\}", INSTANCE_VERSION, services[service]["image"] + ) + assert INSTANCE_IMAGES[service] == built + assert built in checked + + def test_an_image_holding_the_checkout_differs_in_nothing() -> None: listing = "aaa /opt/dpolens/uv.lock\nbbb /opt/dpolens/packages/x.py\n" expected = {"/opt/dpolens/uv.lock": "aaa", "/opt/dpolens/packages/x.py": "bbb"}