diff --git a/evals/trigger/README.md b/evals/trigger/README.md index 8f0b8bc..c725c2b 100644 --- a/evals/trigger/README.md +++ b/evals/trigger/README.md @@ -2,7 +2,8 @@ The retrieval evals measure whether search finds the right clause once an AI assistant asks. This measures whether it asks: how often Claude Code calls DPOLens on its own when the work in front -of it touches personal data, and how often it calls when the work does not. +of it touches personal data, and how often it calls when the work does not. It also runs Qwen Code, +with any model served through an OpenAI-compatible API, such as a small model on your own machine. ## What is here @@ -11,7 +12,7 @@ of it touches personal data, and how often it calls when the work does not. | `tuning.yaml` | 20 tasks used while improving the guidance DPOLens gives an AI assistant | | `held_out.yaml` | 30 tasks that produce the published number and are never used to change anything | | `app/` | Larder, the small TypeScript shop every task runs against | -| `container/` | The container one run happens in | +| `container/` | The container one run happens in, one for each AI assistant | | `../../scripts/run_trigger_test.py` | The runner | ## A task @@ -48,18 +49,25 @@ this compliant" would trigger any AI assistant and the number would mean nothing ## What a run is -Every run is a new container: Claude Code at a pinned version, a fresh checkout of Larder with -its dependencies installed, and an empty profile with DPOLens installed the way +Every run is a new container: the AI assistant at a pinned version (Claude Code 2.1.288 or +Qwen Code 0.27.4), a fresh checkout of Larder with its dependencies installed, and an empty +profile with DPOLens installed. Claude Code gets it the way [Get started](https://dpolens.com/docs/get-started) installs it, from the plugin marketplace, -with `claude plugin configure`. Nothing else is in the -profile: no `CLAUDE.md`, no memory, no other plugins. The runs talk to a throwaway DPOLens -instance built from your checkout, with every pack in it loaded, which is removed afterwards. +with `claude plugin configure`. Qwen Code gets the MCP server on its own, as the entry +`qwen mcp add` writes, beside settings that name the model and its server. Nothing else is in the +profile: no `CLAUDE.md` or `QWEN.md`, no memory, no other plugins. The runs talk to a throwaway +DPOLens instance built from your checkout, with every pack in it loaded, which is removed +afterwards. A run counts as a call when the AI assistant calls `search_policies` or `get_clause`, the two tools that return clause text. It counts as a call before editing when that happens before its -first `Edit`, `Write`, `MultiEdit` or `NotebookEdit`. A run stops as soon as the first call -comes back, since the outcome is known by then. Otherwise it goes until the AI assistant finishes, -25 turns or 10 minutes. +first edit: `Edit`, `Write`, `MultiEdit` or `NotebookEdit` in Claude Code, and `edit`, +`write_file` or `notebook_edit` in Qwen Code. Qwen Code keeps MCP tools out of the prompt until +the model looks one up, so a search can also come through its `tool_call` tool, and counts the +same. A run stops as soon as the first call comes back, since the outcome is known by then. +Otherwise it goes until the AI assistant finishes, 25 turns or 10 minutes. With +`--stop-at-edit`, a run also stops at its first edit, when whether it searched before editing is +known: runs stay short on a slow model, and the batch cannot say what a run would have done after. A run that could not have called DPOLens is retried rather than scored: the plugin not loading, the MCP server not connecting, or the request to the model failing. Each run's calls are also @@ -67,7 +75,7 @@ counted in the instance's query log, and the report names any run where the two ## Running it -You need Docker and either a Claude subscription or an API key. +For Claude Code you need Docker and either a Claude subscription or an API key. ```bash claude setup-token # prints a one-year token for your subscription @@ -93,9 +101,10 @@ of every run, and the full stream of events from each one, which is where to loo did something unexpected. A batch stopped by a usage limit resumes with `--batch `, and `--report ` scores one again without running anything. -`--write held_out-plugin,held_out-mcp-only` writes `evals/results/trigger.json` from held-out -batches, and refuses one that is incomplete, ran on an unpinned model, or disagrees with the -others about the model or the version of Claude Code. +`--write held_out-plugin,held_out-mcp-only` writes `evals/results/trigger.json`, Claude Code's +number, from held-out batches, and refuses one that is incomplete, ran on an unpinned model, +stopped its runs at their first edit, or disagrees with the others about the model or the version +of Claude Code. Building the instance asks the image registries about base images, so a network that cannot reach them stops a batch before it starts. `--reuse-images` starts from the images already @@ -111,6 +120,23 @@ the rule as it now stands to each run's saved stream of events. What was recorde in `outcomes.recorded.jsonl`, and any run that should not have been scored is run again by the next resume. +### With Qwen Code + +Qwen Code talks to any server with an OpenAI-compatible API whose model can call tools, such as +llama.cpp, LM Studio, Ollama, oMLX or vLLM. Name the model as the server lists it, and the server +as the container reaches it: your own machine is `host.docker.internal`. + +```bash +export OPENAI_API_KEY= # any value when it needs none +uv run python scripts/run_trigger_test.py --client qwen-code --set held_out --max-runs 30 \ + --model --base-url http://host.docker.internal:/v1 \ + --sampling temperature=0.6,top_p=0.95,top_k=20 --context-window 262144 --stop-at-edit +``` + +`--sampling` is sent with every request in place of the server's own choice, so give the values +the model's card recommends. `--context-window` tells Qwen Code how much context the server gives +the model. Both are kept with the batch, and resuming it takes the same. + ## Reading the number The report gives two rates: the share of runs that searched on tasks that should, and on tasks diff --git a/evals/trigger/container/Dockerfile b/evals/trigger/container/Dockerfile index d34a34a..fb2ba16 100644 --- a/evals/trigger/container/Dockerfile +++ b/evals/trigger/container/Dockerfile @@ -1,32 +1,24 @@ -# Where one trigger run happens: Claude Code at a pinned version, as an -# ordinary user, in a checkout of the Larder app with its dependencies +# Where one trigger run happens: an AI coding assistant at a pinned version, as +# an ordinary user, in a checkout of the Larder app with its dependencies # installed, the way a developer's machine would have it. # -# Built by scripts/run_trigger_test.py with evals/trigger as the context. +# Built by scripts/run_trigger_test.py with evals/trigger as the context, one +# target for each AI assistant: claude-code or qwen-code. Both start from the +# same checkout. -FROM node:24-bookworm-slim - -ARG CLAUDE_CODE_VERSION=2.1.288 +FROM node:24-bookworm-slim AS larder RUN set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends ca-certificates git openssl; \ - rm -rf /var/lib/apt/lists/*; \ - npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}"; \ - claude --version - -COPY --chmod=755 container/prepare-profile.sh container/run-task.sh /usr/local/bin/ - -# The pinned version is the one measured, so nothing may update it mid-batch. -ENV DISABLE_AUTOUPDATER=1 \ - CLAUDE_CONFIG_DIR=/home/node/.claude + rm -rf /var/lib/apt/lists/* USER node WORKDIR /home/node/larder # A named volume mounted here starts out owned by this user, which is how the # prepared profile reaches every run read-only. -RUN mkdir -p /home/node/.claude /home/node/profile +RUN mkdir -p /home/node/profile COPY --chown=node:node app/ ./ @@ -37,7 +29,7 @@ RUN set -eux; \ npx prisma generate; \ npm run typecheck -# A short history, because a real checkout has one and Claude Code reads it. +# A short history, because a real checkout has one and the AI assistant reads it. RUN set -eux; \ git init -q -b main; \ git config user.name "Sam Okafor"; \ @@ -56,3 +48,40 @@ RUN set -eux; \ git add -A; \ commit "2026-09-19T11:42:00Z" "Orders, with order_placed sent to Segment"; \ test -z "$(git status --porcelain)" + +FROM larder AS claude-code + +ARG CLAUDE_CODE_VERSION=2.1.288 + +USER root +RUN set -eux; \ + npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}"; \ + claude --version + +COPY --chmod=755 container/claude-code/ /usr/local/bin/ + +# The pinned version is the one measured, so nothing may update it mid-batch. +ENV DISABLE_AUTOUPDATER=1 \ + CLAUDE_CONFIG_DIR=/home/node/.claude + +USER node +RUN mkdir -p /home/node/.claude + +FROM larder AS qwen-code + +ARG QWEN_CODE_VERSION=0.27.4 + +USER root +RUN set -eux; \ + npm install -g "@qwen-code/qwen-code@${QWEN_CODE_VERSION}"; \ + qwen --version + +COPY --chmod=755 container/qwen-code/ /usr/local/bin/ + +# Every run approves every tool call, so the notice saying so would only repeat +# itself in each transcript. The pinned version is the one measured: the +# settings each run starts from keep it from updating itself. +ENV QWEN_CODE_SUPPRESS_YOLO_WARNING=1 + +USER node +RUN mkdir -p /home/node/.qwen diff --git a/evals/trigger/container/prepare-profile.sh b/evals/trigger/container/claude-code/prepare-profile.sh similarity index 100% rename from evals/trigger/container/prepare-profile.sh rename to evals/trigger/container/claude-code/prepare-profile.sh diff --git a/evals/trigger/container/run-task.sh b/evals/trigger/container/claude-code/run-task.sh similarity index 100% rename from evals/trigger/container/run-task.sh rename to evals/trigger/container/claude-code/run-task.sh diff --git a/evals/trigger/container/qwen-code/prepare-profile.sh b/evals/trigger/container/qwen-code/prepare-profile.sh new file mode 100644 index 0000000..85156b4 --- /dev/null +++ b/evals/trigger/container/qwen-code/prepare-profile.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Write Qwen Code's settings into an empty profile: the model server it talks +# to, and DPOLens as an MCP server on its own, the entry `qwen mcp add` writes. +# +# prepare-profile.sh mcp-only +# +# The settings arrive on stdin as JSON, so the token is in no argument list. +set -euo pipefail + +setup="${1:?which setup: mcp-only}" +if [[ "$setup" != mcp-only ]]; then + echo "Qwen Code has no DPOLens plugin; the setup is mcp-only" >&2 + exit 2 +fi + +mkdir -p "$HOME/.qwen" +cat > "$HOME/.qwen/settings.json" + +cp -a "$HOME/.qwen"/. /home/node/profile/ diff --git a/evals/trigger/container/qwen-code/run-task.sh b/evals/trigger/container/qwen-code/run-task.sh new file mode 100644 index 0000000..1bc29dc --- /dev/null +++ b/evals/trigger/container/qwen-code/run-task.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# One trigger run in Qwen Code: a fresh copy of the prepared profile, then one prompt. +# +# The prompt and the turn limit arrive as TASK_* variables, and the model is +# named in the settings. The stream of events Qwen Code prints is the whole +# record of the run. +set -euo pipefail + +cp -a /home/node/profile/. "$HOME/.qwen"/ + +exec qwen "${TASK_PROMPT:?}" \ + --output-format stream-json \ + --approval-mode yolo \ + --max-session-turns "${TASK_MAX_TURNS:-25}" diff --git a/scripts/run_trigger_test.py b/scripts/run_trigger_test.py index 4874d35..d3489a3 100644 --- a/scripts/run_trigger_test.py +++ b/scripts/run_trigger_test.py @@ -1,19 +1,21 @@ -"""Measure how often Claude Code calls DPOLens without being told to. +"""Measure how often an AI coding assistant calls DPOLens without being told to. Each task in evals/trigger is something a developer might ask for in the Larder -app. Every run is a new container holding Claude Code, the app, and DPOLens -installed the way Get started on dpolens.com installs it, talking to a -throwaway instance of DPOLens built from this checkout. What is measured is -whether the assistant calls search_policies or get_clause, and whether it does -so before it edits a file. +app. Every run is a new container holding the AI assistant, the app, and +DPOLens installed the way a user installs it, talking to a throwaway instance +of DPOLens built from this checkout. What is measured is whether the AI +assistant calls search_policies or get_clause, and whether it does so before +it edits a file. uv run python scripts/run_trigger_test.py --set tuning # the pilot uv run python scripts/run_trigger_test.py --set held_out --runs 3 --model uv run python scripts/run_trigger_test.py --report # score again -Runs are billed to a Claude subscription through a token from +Claude Code runs are billed to a Claude subscription through a token from `claude setup-token`, read from evals/trigger/.oauth-token, which git ignores, or to an API account when that file is absent and ANTHROPIC_API_KEY is set. +Qwen Code runs talk to the OpenAI-compatible server named by --base-url, with +its key in OPENAI_API_KEY. Everything a batch produces stays in evals/trigger/runs//, also ignored, so a batch stopped by a usage limit resumes where it stopped. """ @@ -32,7 +34,7 @@ import sys import threading import time -from collections.abc import Iterable, Sequence +from collections.abc import Callable, Iterable, Sequence from dataclasses import asdict, dataclass, field from datetime import UTC, datetime from pathlib import Path @@ -50,10 +52,54 @@ TOKEN_FILE = TRIGGER / ".oauth-token" RESULTS = REPO / "evals" / "results" / "trigger.json" -CLAUDE_CODE = "2.1.288" -"""The Claude Code version every run uses, so a batch measures one thing.""" -IMAGE = f"dpolens-trigger-run:{CLAUDE_CODE}" +@dataclass(frozen=True) +class Client: + """An AI coding assistant the test runs, at the version every run of a batch uses, so a + batch measures one thing.""" + + name: str + """Its target in the run container's Dockerfile.""" + label: str + version: str + version_key: str + """Where the first event of its stream gives its version.""" + edits: frozenset[str] + """The tools that change a file.""" + bridge: str | None = None + """A tool that calls the tool named in its input. Qwen Code keeps MCP tools out of the + prompt until the model looks one up, and the model can call one through it as well as by + its own name.""" + turn_cap: str | None = None + """What it prints on reaching the turn limit, when no final event says so.""" + + @property + def image(self) -> str: + return f"dpolens-trigger-run:{self.name}-{self.version}" + + @property + def version_arg(self) -> str: + return self.name.upper().replace("-", "_") + "_VERSION" + + +CLAUDE_CODE = Client( + name="claude-code", + label="Claude Code", + version="2.1.288", + version_key="claude_code_version", + edits=frozenset({"Edit", "Write", "MultiEdit", "NotebookEdit"}), +) +QWEN_CODE = Client( + name="qwen-code", + label="Qwen Code", + version="0.27.4", + version_key="qwen_code_version", + edits=frozenset({"edit", "write_file", "notebook_edit"}), + bridge="tool_call", + turn_cap="Reached max session turns", +) +CLIENTS = {client.name: client for client in (CLAUDE_CODE, QWEN_CODE)} + PROJECT = "dpolens-trigger" INSTANCE_VERSION = "trigger-test" """What the instance's images are built as. Compose names each image after DPOLENS_VERSION, @@ -71,6 +117,18 @@ NETWORK = f"{PROJECT}_default" MCP_URL = "http://mcp:8765/mcp" DEVELOPER = "developer@larder.shop" +OPENAI_KEY = "OPENAI_API_KEY" +"""The variable that holds the model server's key, on this machine and in the container.""" + +SAMPLING: dict[str, Callable[[str], float | int]] = { + "temperature": float, + "top_p": float, + "top_k": int, + "presence_penalty": float, + "frequency_penalty": float, + "repetition_penalty": float, +} +"""The sampling values Qwen Code sends with every request when its settings name them.""" SETS = ("tuning", "held_out") SETUPS = ("plugin", "mcp-only") @@ -80,7 +138,6 @@ COUNTED = ("search_policies", "get_clause") """The tools that return clause text. Listing documents is not consulting them.""" -EDITS = frozenset({"Edit", "Write", "MultiEdit", "NotebookEdit"}) VERBS = ("collect", "store", "log", "share", "delete") KINDS = ("coding", "question") @@ -216,6 +273,7 @@ class Observation: prefix: str server: str + client: Client = CLAUDE_CODE model: str | None = None version: str | None = None server_status: str | None = None @@ -225,6 +283,7 @@ class Observation: first_call: int | None = None """Where in `tools` the first counted call is.""" first_edit: int | None = None + turn_capped: bool = False call_answered: bool = False call_failed: bool = False """Whether the first counted call came back as an error, which no setup should cause.""" @@ -237,6 +296,18 @@ class Observation: """Whether the last thing that happened was a failed request to the model.""" _names: dict[str, str] = field(default_factory=dict) _call_id: str | None = None + _registered: tuple[str, ...] = () + + def read(self, line: str) -> None: + """One line of a run's output: an event, or text printed outside the events.""" + try: + event = json.loads(line) + except json.JSONDecodeError: + if self.client.turn_cap and line.startswith(self.client.turn_cap): + self.turn_capped = True + return + if isinstance(event, dict): + self.see(event) def see(self, event: dict[str, Any]) -> None: kind, subtype = event.get("type"), event.get("subtype") @@ -256,7 +327,8 @@ def see(self, event: dict[str, Any]) -> None: def _init(self, event: dict[str, Any]) -> None: self.started = True self.model = event.get("model") - self.version = event.get("claude_code_version") + self.version = event.get(self.client.version_key) + self._registered = tuple(str(name) for name in event.get("tools") or []) for server in event.get("mcp_servers") or []: if isinstance(server, dict) and server.get("name") == self.server: self.server_status = server.get("status") @@ -274,15 +346,28 @@ def _assistant(self, event: dict[str, Any]) -> None: for block in blocks(event): if block.get("type") != "tool_use": continue - name = str(block.get("name", "")) + name = self._target(block) self._names[str(block.get("id"))] = name self.tools.append(name) if self.first_call is None and self.counts(name): self.first_call = len(self.tools) - 1 self._call_id = str(block.get("id")) - if self.first_edit is None and name in EDITS: + if self.first_edit is None and name in self.client.edits: self.first_edit = len(self.tools) - 1 + def _target(self, block: dict[str, Any]) -> str: + """The tool a call runs. Through the bridge, it is the tool named, resolved the way + Qwen Code resolves it: the exact name, or else the only one differing in case.""" + name = str(block.get("name", "")) + given = block.get("input") + named = given.get("name") if isinstance(given, dict) else None + if name != self.client.bridge or not isinstance(named, str): + return name + if named in self._registered: + return named + same = {tool for tool in self._registered if tool.lower() == named.lower()} + return same.pop() if len(same) == 1 else named + def _user(self, event: dict[str, Any]) -> None: for block in blocks(event): if block.get("type") != "tool_result": @@ -310,7 +395,7 @@ def before_edit(self) -> bool: def unusable(self, setup: str) -> str | None: """Why this run could not have called DPOLens, if it could not.""" if not self.started: - return "Claude Code never started a session" + return f"{self.client.label} never started a session" if setup == "plugin" and ("dpolens" not in self.plugins or self.plugin_errors): return f"the plugin did not load: {'; '.join(self.plugin_errors) or 'not listed'}" if self.server_status not in ("connected", "pending"): @@ -320,9 +405,27 @@ def unusable(self, setup: str) -> str | None: if self.retrying: return f"the model requests kept failing ({', '.join(sorted(set(self.retries)))})" if self.result and self.result.get("is_error") and capped_by(self.result) is None: - return f"the run failed: {self.result.get('result') or self.result.get('subtype')}" + error = self.result.get("error") + reason = ( + self.result.get("result") + or (error.get("message") if isinstance(error, dict) else None) + or self.result.get("subtype") + ) + return f"the run failed: {reason}" return None + @property + def capped(self) -> str | None: + """Which limit ended the run, if one did.""" + return "turns" if self.turn_capped else capped_by(self.result or {}) + + def settled(self, stop_at_edit: bool) -> bool: + """Whether the run's outcome is known: its first call came back or, when runs stop at + the first edit, it edited before calling.""" + return self.call_answered or ( + stop_at_edit and self.first_edit is not None and self.first_call is None + ) + def capped_by(result: dict[str, Any]) -> str | None: """Which limit ended a run, read from Claude Code's final event.""" @@ -359,6 +462,8 @@ class Outcome: search_failed: bool = False """Whether the search that decided the run came back as an error. Batches recorded before this was noted read as False.""" + stopped_at_edit: bool = False + """Whether the run was ended at its first edit, which only `--stop-at-edit` does.""" def outcome_of( @@ -372,10 +477,11 @@ def outcome_of( clock: bool, seconds: float, rows: int | None, + stopped_at_edit: bool = False, ) -> Outcome: unusable = seen.unusable(setup) result = seen.result or {} - capped = "clock" if clock else capped_by(result) if not seen.called else None + capped = "clock" if clock else seen.capped if not seen.called else None return Outcome( task=task.id, run=run, @@ -396,6 +502,7 @@ def outcome_of( dpolens_calls=seen.dpolens_answered, query_log_rows=rows, search_failed=seen.call_failed, + stopped_at_edit=stopped_at_edit, ) @@ -501,7 +608,7 @@ def runs_of(task: Task) -> list[Outcome]: } -def print_score(found: dict[str, Any]) -> None: +def print_score(found: dict[str, Any], client: Client) -> None: labels = { "should": "coding tasks that should search", "should_not": "coding tasks that should not", @@ -531,9 +638,10 @@ def print_score(found: dict[str, Any]) -> None: f" runs scored {found['runs_scored']}, capped {found['capped']}, " f"attempts not scored {found['attempts_not_scored']}" ) + cost = found["estimated_cost_usd"] print( - f" estimated cost {found['estimated_cost_usd']} USD, " - f"model {', '.join(found['models'])}, Claude Code {', '.join(found['versions'])}" + (f" estimated cost {cost} USD, " if cost else " ") + + f"model {', '.join(found['models'])}, {client.label} {', '.join(found['versions'])}" ) for key, label in ( ("never_searched", "never searched"), @@ -729,15 +837,17 @@ def stale_images() -> list[str]: return stale -def build_image() -> None: - print(f"building the run container with Claude Code {CLAUDE_CODE}", flush=True) +def build_image(client: Client) -> None: + print(f"building the run container with {client.label} {client.version}", flush=True) docker( "build", "--quiet", + "--target", + client.name, "-t", - IMAGE, + client.image, "--build-arg", - f"CLAUDE_CODE_VERSION={CLAUDE_CODE}", + f"{client.version_arg}={client.version}", "-f", str(TRIGGER / "container" / "Dockerfile"), str(TRIGGER), @@ -755,10 +865,48 @@ def marketplace(batch: Path) -> Path: return target -def prepare_profile(setup: str, token: str, volume: str, plugin_source: Path) -> None: +def qwen_settings( + model: str, + base_url: str, + token: str, + sampling: dict[str, float | int] | None, + context_window: int | None, +) -> dict[str, Any]: + """Qwen Code's settings for a batch: the model at its server, and DPOLens as the entry + `qwen mcp add` writes. Nothing reports usage, fetches a model catalogue or updates + Qwen Code, so a run talks to the model server and DPOLens only.""" + generation: dict[str, Any] = {} + if sampling: + generation["samplingParams"] = sampling + if context_window: + generation["contextWindowSize"] = context_window + provider = { + "id": model, + "name": model, + "baseUrl": base_url, + "envKey": OPENAI_KEY, + "generationConfig": generation, + } + return { + "general": {"enableAutoUpdate": False}, + "privacy": {"usageStatisticsEnabled": False}, + "security": {"auth": {"selectedType": "openai"}}, + "model": {"name": model, "catalogRefresh": False}, + "modelProviders": {"openai": [provider]}, + "mcpServers": { + "dpolens": {"httpUrl": MCP_URL, "headers": {"Authorization": f"Bearer {token}"}} + }, + } + + +def prepare_profile( + client: Client, args: argparse.Namespace, token: str, volume: str, plugin_source: Path +) -> None: """Install DPOLens into an empty profile once, for every run of the batch to copy.""" values: dict[str, Any] - if setup == "plugin": + if client is QWEN_CODE: + values = qwen_settings(args.model, args.base_url, token, args.sampling, args.context_window) + elif args.setup == "plugin": values = {"url": MCP_URL, "token": token} else: values = {"type": "http", "url": MCP_URL, "headers": {"Authorization": f"Bearer {token}"}} @@ -773,9 +921,9 @@ def prepare_profile(setup: str, token: str, volume: str, plugin_source: Path) -> f"{volume}:/home/node/profile", "-v", f"{plugin_source}:/marketplace:ro", - IMAGE, + client.image, "prepare-profile.sh", - setup, + args.setup, input=json.dumps(values), capture_output=True, ) @@ -789,6 +937,7 @@ def read_lines(stream: Any, lines: queue.Queue[str | None]) -> None: def run_once( task: Task, + client: Client, setup: str, model: str | None, budget: float | None, @@ -796,8 +945,10 @@ def run_once( plugin_source: Path, credential: tuple[str, str], transcript: Path, -) -> tuple[Observation, bool, bool, float]: - """One container, one prompt. Returns what was seen, and why it ended.""" + stop_at_edit: bool = False, +) -> tuple[Observation, bool, bool, bool, float]: + """One container, one prompt. Returns what was seen, and why it ended: at the first call + answered, at the first edit, or on the clock.""" name = f"{PROJECT}-run-{secrets.token_hex(4)}" env = { **os.environ, @@ -829,12 +980,15 @@ def run_once( f"{volume}:/home/node/profile:ro", "-v", f"{plugin_source}:/marketplace:ro", - IMAGE, + # A model server on this machine is host.docker.internal, a name Docker Desktop gives + # every container and Docker on Linux has to be told. + *(["--add-host", "host.docker.internal:host-gateway"] if client is QWEN_CODE else []), + client.image, "run-task.sh", ] - seen = Observation(prefix=PREFIXES[setup], server=SERVERS[setup]) + seen = Observation(prefix=PREFIXES[setup], server=SERVERS[setup], client=client) started = time.monotonic() - stopped_at_call = clock = False + stopped_at_call = stopped_at_edit = clock = False process = subprocess.Popen( command, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True @@ -856,20 +1010,16 @@ def run_once( if line is None: break record.write(line) - try: - event = json.loads(line) - except json.JSONDecodeError: - continue - if isinstance(event, dict): - seen.see(event) - if seen.call_answered: - stopped_at_call = True + seen.read(line) + if seen.settled(stop_at_edit): + stopped_at_call = seen.call_answered + stopped_at_edit = not stopped_at_call break - if stopped_at_call or clock: + if stopped_at_call or stopped_at_edit or clock: subprocess.run(["docker", "kill", name], capture_output=True, check=False) process.wait(timeout=60) - return seen, stopped_at_call, clock, time.monotonic() - started + return seen, stopped_at_call, stopped_at_edit, clock, time.monotonic() - started def attempt_run( @@ -884,8 +1034,9 @@ def attempt_run( batch: Path, ) -> Outcome: before = instance.rows() - seen, at_call, clock, seconds = run_once( + seen, at_call, at_edit, clock, seconds = run_once( task, + CLIENTS[args.client], args.setup, args.model, args.max_budget_usd, @@ -893,6 +1044,7 @@ def attempt_run( plugin_source, credential, batch / "transcripts" / f"{task.id}.{run}.{attempt}.jsonl", + args.stop_at_edit, ) after = instance.rows() rows = after - before if before is not None and after is not None else None @@ -906,6 +1058,7 @@ def attempt_run( clock=clock, seconds=seconds, rows=rows, + stopped_at_edit=at_edit, ) @@ -914,6 +1067,8 @@ def verdict(outcome: Outcome) -> str: return f"not scored, {outcome.unusable}" if outcome.called: return "searched" + ("" if outcome.before_edit else " after editing") + if outcome.stopped_at_edit: + return f"edited without searching, {outcome.seconds:.0f}s" return f"did not search ({outcome.capped or 'finished'}), {outcome.seconds:.0f}s" @@ -927,8 +1082,11 @@ def read_outcomes(path: Path) -> list[Outcome]: ] -def credential_for_runs() -> tuple[str, str] | None: - """The variable Claude Code authenticates with, and its value.""" +def credential_for_runs(client: Client) -> tuple[str, str] | None: + """The variable the AI assistant authenticates with, and its value.""" + if client is QWEN_CODE: + key = os.environ.get(OPENAI_KEY) + return (OPENAI_KEY, key) if key else None if TOKEN_FILE.is_file(): return "CLAUDE_CODE_OAUTH_TOKEN", TOKEN_FILE.read_text(encoding="utf-8").strip() if os.environ.get("ANTHROPIC_API_KEY"): @@ -936,28 +1094,57 @@ def credential_for_runs() -> tuple[str, str] | None: return None +def refusal(args: argparse.Namespace) -> str | None: + """Why the options cannot make a batch, when they cannot.""" + client = CLIENTS[args.client] + if client is QWEN_CODE: + if args.setup == "plugin": + return "the plugin is Claude Code's: Qwen Code runs use --setup mcp-only" + if not args.model or not args.base_url: + return "Qwen Code runs need --model and --base-url: the model, at the server holding it" + if args.max_budget_usd is not None: + return "--max-budget-usd is Claude Code's limit, and Qwen Code has none" + elif args.base_url or args.sampling or args.context_window: + return "--base-url, --sampling and --context-window are for Qwen Code runs" + if args.set == "held_out" and not args.model: + return "a held-out batch is published, so its model is pinned: pass --model" + return None + + def run_batch(args: argparse.Namespace) -> int: + client = CLIENTS[args.client] + args.setup = args.setup or ("plugin" if client is CLAUDE_CODE else "mcp-only") + refused = refusal(args) + if refused: + print(refused) + return 2 + tasks = read_tasks(TRIGGER / f"{args.set}.yaml") if args.tasks: wanted = set(args.tasks.split(",")) tasks = [task for task in tasks if task.id in wanted] - if args.set == "held_out" and not args.model: - print("a held-out batch is published, so its model is pinned: pass --model") - return 2 planned = len(tasks) * args.runs if planned > args.max_runs: print(f"{planned} runs planned, more than --max-runs {args.max_runs}; nothing started") return 2 - credential = credential_for_runs() + credential = credential_for_runs(client) if credential is None: print( - f"no subscription token at {TOKEN_FILE}: run `claude setup-token` and save it " + f"set {OPENAI_KEY} to the key of the server at --base-url, or to any value when " + "it needs none" + if client is QWEN_CODE + else f"no subscription token at {TOKEN_FILE}: run `claude setup-token` and save it " "there, or set ANTHROPIC_API_KEY to bill an API account instead" ) return 2 - batch_id = args.batch or f"{args.set}-{args.setup}-{datetime.now(UTC):%Y%m%d-%H%M}" + named = ( + args.setup + if client is CLAUDE_CODE + else f"{client.name}-{re.sub(r'[^A-Za-z0-9._-]+', '-', args.model)}" + ) + batch_id = args.batch or f"{args.set}-{named}-{datetime.now(UTC):%Y%m%d-%H%M}" batch = RUNS / batch_id (batch / "transcripts").mkdir(parents=True, exist_ok=True) outcomes_file = batch / "outcomes.jsonl" @@ -972,38 +1159,40 @@ def run_batch(args: argparse.Namespace) -> int: else "built from the checkout" ), } + asked = conditions(args) if meta_file.is_file(): meta = json.loads(meta_file.read_text(encoding="utf-8")) - asked = {"set": args.set, "setup": args.setup, "model": args.model} - if any(meta.get(key) != value for key, value in asked.items()): + differ = [key for key, value in asked.items() if recorded(meta, key) != value] + if differ: print( - f"batch {batch_id} is {meta['set']}, {meta['setup']}, {meta['model']}: " - "resume it with those" + f"batch {batch_id} was started with other {', '.join(differ)}: resume it with those" ) return 2 meta.setdefault("resumed", []).append(this_run) else: meta = { - "set": args.set, - "setup": args.setup, - "model": args.model, - "claude_code": CLAUDE_CODE, + **{key: value for key, value in asked.items() if value is not None or key == "model"}, + "client_version": client.version, + **({"claude_code": client.version} if client is CLAUDE_CODE else {}), "runs_per_task": args.runs, "max_turns": MAX_TURNS, "max_budget_usd": args.max_budget_usd, **this_run, } meta_file.write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8") - print(f"batch {batch_id}: {len(tasks)} tasks, {args.runs} runs each, setup {args.setup}") + print( + f"batch {batch_id}: {len(tasks)} tasks, {args.runs} runs each, " + f"{client.label}, setup {args.setup}" + ) - build_image() + build_image(client) instance = Instance() - volume = f"{PROJECT}-profile-{args.setup}" + volume = f"{PROJECT}-profile-{client.name}-{args.setup}" failures = 0 try: instance.start(build=not args.reuse_images) plugin_source = marketplace(batch) - prepare_profile(args.setup, instance.token, volume, plugin_source) + prepare_profile(client, args, instance.token, volume, plugin_source) for task in tasks: for run in range(1, args.runs + 1): @@ -1040,10 +1229,30 @@ def run_batch(args: argparse.Namespace) -> int: found = score(tasks, outcomes) (batch / "score.json").write_text(json.dumps(found, indent=2) + "\n", encoding="utf-8") - print_score(found) + print_score(found, client) return 0 +def conditions(args: argparse.Namespace) -> dict[str, Any]: + """What a batch measures under, which every run of it shares, resumed ones included.""" + return { + "set": args.set, + "setup": args.setup, + "model": args.model, + "client": args.client, + "base_url": args.base_url, + "sampling": args.sampling, + "context_window": args.context_window, + "stop_at_edit": args.stop_at_edit or None, + } + + +def recorded(meta: dict[str, Any], key: str) -> Any: + """A condition as a batch recorded it. Batches from before there was a choice of client + ran Claude Code.""" + return meta.get(key, CLAUDE_CODE.name if key == "client" else None) + + def checkout() -> dict[str, Any]: """The commit a batch ran from, and whether tracked files had changed since.""" @@ -1070,22 +1279,18 @@ def reread(batch_id: str) -> int: meta = json.loads((batch / "batch.json").read_text(encoding="utf-8")) tasks = {task.id: task for task in read_tasks(TRIGGER / f"{meta['set']}.yaml")} outcomes_file = batch / "outcomes.jsonl" - recorded = read_outcomes(outcomes_file) + outcomes = read_outcomes(outcomes_file) first = batch / "outcomes.recorded.jsonl" if not first.is_file(): shutil.copy2(outcomes_file, first) + client = CLIENTS[recorded(meta, "client")] updated = [] - for old in recorded: - seen = Observation(prefix=PREFIXES[old.setup], server=SERVERS[old.setup]) + for old in outcomes: + seen = Observation(prefix=PREFIXES[old.setup], server=SERVERS[old.setup], client=client) transcript = batch / "transcripts" / f"{old.task}.{old.run}.{old.attempt}.jsonl" for line in transcript.read_text(encoding="utf-8").splitlines(): - try: - event = json.loads(line) - except json.JSONDecodeError: - continue - if isinstance(event, dict): - seen.see(event) + seen.read(line) new = outcome_of( tasks[old.task], old.run, @@ -1096,6 +1301,7 @@ def reread(batch_id: str) -> int: clock=old.capped == "clock", seconds=old.seconds, rows=old.query_log_rows, + stopped_at_edit=old.stopped_at_edit, ) if verdict(new) != verdict(old): print( @@ -1121,6 +1327,14 @@ def published(batch_ids: Sequence[str]) -> dict[str, Any]: meta = json.loads((RUNS / batch_id / "batch.json").read_text(encoding="utf-8")) if meta["set"] != "held_out": raise NotPublishable(f"{batch_id} is a {meta['set']} batch; only held-out is published") + if recorded(meta, "client") != CLAUDE_CODE.name: + raise NotPublishable( + f"{batch_id} ran {meta['client']}; {RESULTS.name} is Claude Code's" + ) + if meta.get("stop_at_edit"): + raise NotPublishable( + f"{batch_id} stopped runs at their first edit, so it cannot say what they did after" + ) if not meta.get("model"): raise NotPublishable( f"{batch_id} ran on whatever model was the default, not a pinned one" @@ -1176,19 +1390,55 @@ def report(batch_id: str) -> int: meta = json.loads((batch / "batch.json").read_text(encoding="utf-8")) tasks = read_tasks(TRIGGER / f"{meta['set']}.yaml") found = score(tasks, read_outcomes(batch / "outcomes.jsonl")) - print(f"batch {batch_id}: {meta['set']}, setup {meta['setup']}") - print_score(found) + client = CLIENTS[recorded(meta, "client")] + print(f"batch {batch_id}: {meta['set']}, {client.label}, setup {meta['setup']}") + print_score(found, client) return 0 -def main() -> int: +def sampling(text: str) -> dict[str, float | int]: + """`temperature=0.6,top_k=20` as the values Qwen Code sends with every request.""" + found: dict[str, float | int] = {} + for pair in text.split(","): + key, _, value = (part.strip() for part in pair.partition("=")) + if key not in SAMPLING: + raise argparse.ArgumentTypeError(f"{key!r} is not one of {', '.join(SAMPLING)}") + try: + found[key] = SAMPLING[key](value) + except ValueError: + raise argparse.ArgumentTypeError(f"{key} takes a number, not {value!r}") from None + return found + + +def arguments(argv: Sequence[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) parser.add_argument("--set", choices=SETS, default="tuning") - parser.add_argument("--setup", choices=SETUPS, default="plugin") + parser.add_argument( + "--client", choices=sorted(CLIENTS), default=CLAUDE_CODE.name, help="the AI assistant" + ) + parser.add_argument( + "--setup", choices=SETUPS, help="plugin, Claude Code's default, or the MCP server alone" + ) parser.add_argument("--runs", type=int, default=1, help="runs of each task") parser.add_argument("--model", help="the full model id; Claude Code's default when left out") + parser.add_argument( + "--base-url", help="Qwen Code: the OpenAI-compatible server, as the container reaches it" + ) + parser.add_argument( + "--sampling", + type=sampling, + help="Qwen Code: sampling sent with every request, as temperature=0.6,top_k=20", + ) + parser.add_argument( + "--context-window", type=int, help="Qwen Code: the context the server gives the model" + ) + parser.add_argument( + "--stop-at-edit", + action="store_true", + help="end a run at its first edit, when whether it searched before editing is known", + ) parser.add_argument("--tasks", help="only these task ids, separated by commas") parser.add_argument("--batch", help="a batch to resume, or the name for a new one") parser.add_argument( @@ -1212,7 +1462,11 @@ def main() -> int: metavar="BATCHES", help="write evals/results/trigger.json from held-out batches, separated by commas", ) - args = parser.parse_args() + return parser.parse_args(argv) + + +def main() -> int: + args = arguments() if args.report: return report(args.report) diff --git a/tests/unit/test_trigger_runs.py b/tests/unit/test_trigger_runs.py index 521cefe..db52a95 100644 --- a/tests/unit/test_trigger_runs.py +++ b/tests/unit/test_trigger_runs.py @@ -1,12 +1,13 @@ -"""Reading a trigger run from Claude Code's stream of events, and scoring a batch. +"""Reading a trigger run from an AI assistant's stream of events, and scoring a batch. -The events are built by hand in the shape Claude Code prints with +The events are built by hand in the shapes Claude Code and Qwen Code print with `--output-format stream-json`, so these tests pin what counts as a call -without running an assistant. +without running an AI assistant. """ from __future__ import annotations +import argparse import json import re import subprocess @@ -24,10 +25,13 @@ import run_trigger_test # noqa: E402 from run_trigger_test import ( # noqa: E402 BAKED_IN, + CLIENTS, INSTANCE_IMAGES, INSTANCE_VERSION, LAWS, + MCP_URL, PREFIXES, + QWEN_CODE, SERVERS, TRIGGER, Instance, @@ -36,13 +40,18 @@ Outcome, Task, agrees, + arguments, differing, digests, outcome_of, published, + qwen_settings, read_outcomes, read_tasks, reread, + run_batch, + run_once, + sampling, score, token_from, ) @@ -427,6 +436,7 @@ def held_out_batch( model: str | None = "claude-sonnet-5-5", drop: int = 0, failed: bool = False, + conditions: dict[str, Any] | None = None, ) -> None: """A batch in which every task did what its label says, once.""" batch = runs / name @@ -438,6 +448,7 @@ def held_out_batch( "claude_code": "2.1.288", "runs_per_task": 1, "started": "2026-10-09T08:00:00+00:00", + **(conditions or {}), } (batch / "batch.json").write_text(json.dumps(meta)) session = {**init(), "model": model or "claude-sonnet-5-5"} @@ -484,6 +495,8 @@ def test_a_complete_pinned_held_out_batch_is_published( ({"model": None}, "not a pinned one"), ({"drop": 1}, "scored 29 of 30 runs"), ({"failed": True}, "came back as an error in 15 runs"), + ({"conditions": {"stop_at_edit": True}}, "cannot say what they did after"), + ({"conditions": {"client": "qwen-code"}}, "is Claude Code's"), ], ) def test_a_batch_that_cannot_stand_behind_the_number_is_refused( @@ -505,3 +518,226 @@ def test_batches_on_different_models_are_not_published_together( with pytest.raises(NotPublishable, match="different models"): published(["one", "two"]) + + +QWEN_SEARCH = PREFIXES["mcp-only"] + "search_policies" + + +def qwen_init(status: str = "connected") -> dict[str, Any]: + """The first event of a Qwen Code 0.27.4 run, which lists every tool, deferred ones too.""" + return { + "type": "system", + "subtype": "init", + "model": "Qwen3.5-4B-8bit", + "qwen_code_version": "0.27.4", + "mcp_servers": [{"name": SERVERS["mcp-only"], "status": status}], + "tools": ["read_file", "edit", "write_file", "tool_search", "tool_call", QWEN_SEARCH], + } + + +def bridged(tool_id: str, name: str) -> dict[str, Any]: + """A call through Qwen Code's bridge to a tool kept out of the prompt.""" + return { + "type": "assistant", + "message": { + "content": [ + { + "type": "tool_use", + "id": tool_id, + "name": "tool_call", + "input": {"name": name, "arguments": {"query": "logging IP addresses"}}, + } + ] + }, + } + + +def qwen(*lines: dict[str, Any] | str) -> Observation: + seen = Observation(prefix=PREFIXES["mcp-only"], server=SERVERS["mcp-only"], client=QWEN_CODE) + for line in lines: + seen.read(line if isinstance(line, str) else json.dumps(line)) + return seen + + +def test_a_search_through_qwen_codes_bridge_is_a_call() -> None: + """Qwen Code keeps MCP tools out of the prompt, so its searches arrive as tool_call.""" + seen = qwen( + qwen_init(), + use("1", "tool_search"), + answer("1"), + bridged("2", QWEN_SEARCH), + answer("2"), + use("3", "edit"), + ) + + assert seen.called and seen.before_edit and seen.call_answered + assert seen.version == "0.27.4" + assert seen.unusable("mcp-only") is None + assert seen.dpolens_answered == 1 + + +def test_looking_a_tool_up_is_not_calling_it() -> None: + seen = qwen(qwen_init(), use("1", "tool_search"), answer("1"), use("2", "write_file")) + + assert not seen.called + assert seen.first_edit == 1 + + +def test_a_bridged_name_resolves_the_way_qwen_code_resolves_it() -> None: + """The exact name, or the one tool differing only in case. A bare tool name reaches no + server, since Qwen Code refuses it, so it is not a call.""" + assert qwen(qwen_init(), bridged("1", QWEN_SEARCH.upper())).called + assert not qwen(qwen_init(), bridged("1", "search_policies")).called + + +def test_qwen_codes_own_edit_tools_are_edits() -> None: + seen = qwen(qwen_init(), use("1", "write_file"), bridged("2", QWEN_SEARCH), answer("2")) + + assert seen.called + assert not seen.before_edit + + +def test_a_run_qwen_code_ended_at_the_turn_limit_is_scored_as_capped() -> None: + """Qwen Code says so on stderr and exits, with no final event.""" + seen = qwen( + qwen_init(), + use("1", "read_file"), + answer("1"), + "Reached max session turns for this session. Increase the number of turns by " + "specifying maxSessionTurns in settings.json.\n", + ) + outcome = outcome_of( + TASKS[2], 1, 1, "mcp-only", seen, stopped_at_call=False, clock=False, seconds=1.0, rows=0 + ) + + assert outcome.scored + assert outcome.capped == "turns" + + +def test_a_qwen_code_run_whose_model_request_failed_is_not_scored() -> None: + seen = qwen( + qwen_init(), + { + "type": "result", + "subtype": "error_during_execution", + "is_error": True, + "error": {"message": "[API Error: 401 Invalid API key]"}, + }, + ) + + assert seen.unusable("mcp-only") == "the run failed: [API Error: 401 Invalid API key]" + + +class Finished: + """A stand-in for the container: its whole output at once.""" + + def __init__(self, lines: list[str]) -> None: + self.stdout = iter(lines) + + def wait(self, timeout: float | None = None) -> int: + return 0 + + +@pytest.mark.parametrize("stop_at_edit", [True, False]) +def test_stop_at_edit_ends_a_run_at_its_first_edit( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, stop_at_edit: bool +) -> None: + """Whether a run searched before editing is known at its first edit.""" + events = [qwen_init(), use("1", "edit"), answer("1"), bridged("2", QWEN_SEARCH), answer("2")] + killed: list[list[str]] = [] + + def kill(command: list[str], **_: Any) -> subprocess.CompletedProcess[str]: + killed.append(command) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr( + subprocess, "Popen", lambda *_, **__: Finished([json.dumps(e) + "\n" for e in events]) + ) + monkeypatch.setattr(subprocess, "run", kill) + + seen, at_call, at_edit, clock, _ = run_once( + TASKS[0], + QWEN_CODE, + "mcp-only", + "Qwen3.5-4B-8bit", + None, + "volume", + tmp_path, + ("OPENAI_API_KEY", "key"), + tmp_path / "transcript.jsonl", + stop_at_edit, + ) + + assert (at_edit, at_call, seen.called) == ( + (True, False, False) if stop_at_edit else (False, True, True) + ) + assert not clock + assert killed and killed[0][:2] == ["docker", "kill"] + + +def test_qwen_codes_settings_name_the_server_the_model_and_dpolens() -> None: + settings = qwen_settings( + "Qwen3.5-4B-8bit", + "http://host.docker.internal:8181/v1", + "dpol_token", + {"temperature": 0.6, "top_k": 20}, + 262144, + ) + + assert settings["mcpServers"] == { + "dpolens": {"httpUrl": MCP_URL, "headers": {"Authorization": "Bearer dpol_token"}} + } + [provider] = settings["modelProviders"]["openai"] + assert provider["baseUrl"] == "http://host.docker.internal:8181/v1" + assert provider["envKey"] == "OPENAI_API_KEY" + assert provider["generationConfig"] == { + "samplingParams": {"temperature": 0.6, "top_k": 20}, + "contextWindowSize": 262144, + } + assert settings["model"] == {"name": "Qwen3.5-4B-8bit", "catalogRefresh": False} + assert settings["privacy"]["usageStatisticsEnabled"] is False + assert settings["general"]["enableAutoUpdate"] is False + + +def test_sampling_is_read_as_the_numbers_qwen_code_sends() -> None: + assert sampling("temperature=0.6, top_p=0.95,top_k=20") == { + "temperature": 0.6, + "top_p": 0.95, + "top_k": 20, + } + with pytest.raises(argparse.ArgumentTypeError, match="min_p"): + sampling("min_p=0") + + +@pytest.mark.parametrize( + ("options", "reason"), + [ + (["--client", "qwen-code", "--model", "m"], "need --model and --base-url"), + (["--client", "qwen-code", "--setup", "plugin"], "the plugin is Claude Code's"), + (["--base-url", "http://host.docker.internal:8181/v1"], "are for Qwen Code runs"), + (["--set", "held_out", "--client", "qwen-code", "--base-url", "u"], "need --model"), + ], +) +def test_options_that_cannot_make_a_batch_start_nothing( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + options: list[str], + reason: str, +) -> None: + monkeypatch.setattr(run_trigger_test, "RUNS", tmp_path) + + assert run_batch(arguments(options)) == 2 + assert reason in capsys.readouterr().out + assert not any(tmp_path.iterdir()) + + +def test_every_client_has_a_target_and_its_two_scripts() -> None: + container = TRIGGER / "container" + dockerfile = (container / "Dockerfile").read_text(encoding="utf-8") + targets = re.findall(r"^FROM \S+ AS (\S+)$", dockerfile, re.MULTILINE) + + for name in CLIENTS: + assert name in targets + assert (container / name / "prepare-profile.sh").is_file() + assert (container / name / "run-task.sh").is_file()