From c9ffaca52864d939be2caced124f27963cdb895c Mon Sep 17 00:00:00 2001 From: Fengzdadi <453788063@qq.com> Date: Wed, 23 Sep 2026 00:09:04 -0400 Subject: [PATCH 1/4] fix(evaluation): register a Scope for each arm before running Codex Since #1401 the Server generates Scope IDs and rejects an explicit Scope that does not exist. The console still passed the literal `eval:{run_id}:{arm}` to the Codex plugin, so its UserPromptSubmit hook got a 404 while resolving the Scope and exited without capturing or recalling. The ON arm then behaved like the OFF arm and failed treatment validation. Each arm now creates its Scope through `POST /v1/scopes` once the Server is ready, using `eval:{run_id}:{arm}` as the idempotency key, and passes the returned ID to Codex. The evidence query and treatment validation use that ID, and the evidence records the key as `scope_key` so reports can tie it to the run. Evidence recorded before this change carries the key as its Scope ID and still validates. Co-Authored-By: Claude Opus 5.5 --- evaluation/src/powercontext_eval/models.py | 6 + .../src/powercontext_eval/powercontext_sut.py | 68 +++++++++-- .../src/powercontext_eval/web/models.py | 1 + .../src/powercontext_eval/web/reporting.py | 5 +- .../tests/contract/test_codex_contract.py | 107 ++++++++++++++---- evaluation/tests/web/test_reporting.py | 16 +++ 6 files changed, 169 insertions(+), 34 deletions(-) diff --git a/evaluation/src/powercontext_eval/models.py b/evaluation/src/powercontext_eval/models.py index da476af90b..e34b3abd0e 100644 --- a/evaluation/src/powercontext_eval/models.py +++ b/evaluation/src/powercontext_eval/models.py @@ -28,6 +28,12 @@ class Arm(StrEnum): ON = "on" +def scope_key(run_id: str, arm: Arm) -> str: + """Return the key that identifies the Scope registered for one run arm.""" + + return f"eval:{run_id}:{arm.value}" + + class TreatmentMode(StrEnum): """The exact treatment arms executed for one evaluation task.""" diff --git a/evaluation/src/powercontext_eval/powercontext_sut.py b/evaluation/src/powercontext_eval/powercontext_sut.py index 239ea544ac..05163886c0 100644 --- a/evaluation/src/powercontext_eval/powercontext_sut.py +++ b/evaluation/src/powercontext_eval/powercontext_sut.py @@ -67,7 +67,7 @@ is_safe_codex_model, ) from powercontext_eval.errors import CommandError, CommandFailed, CommandTimedOut, PowerContextEvalError -from powercontext_eval.models import Arm +from powercontext_eval.models import Arm, scope_key from powercontext_eval.process import CommandResult, ProcessRunner from powercontext_eval.tokensflow import ( DrainDeadline, @@ -141,6 +141,27 @@ code = 11 sys.exit(code) """.strip() +# The Server generates Scope IDs and rejects an explicit Scope that does not exist, +# so each arm registers its own Scope and hands the returned ID to Codex. +_SCOPE_CREATION_SCRIPT = """ +import json +import sys +from urllib.request import Request, urlopen + +body = { + "title": "Evaluation arm", + "summary": "Isolated Scope for one evaluation arm.", + "idempotency_key": sys.argv[1], +} +request = Request( + "http://127.0.0.1:8000/v1/scopes", + data=json.dumps(body).encode("utf-8"), + headers={"Accept": "application/json", "Content-Type": "application/json"}, + method="POST", +) +with urlopen(request, timeout=10) as response: + print(json.dumps({"scope_id": json.load(response)["scope_id"]})) +""".strip() LOOPBACK_NO_PROXY = "127.0.0.1,localhost,::1" _TOKENSFLOW_RETRY_ATTEMPTS = 10 _PLUGIN_RELATIVE = Path("integrations/codex/plugins/powercontext") @@ -394,6 +415,9 @@ class TreatmentEvidence: prompt_sources: int mcp_requests: int scope_id: str + # Identifies the run and arm that registered ``scope_id``; absent in evidence + # recorded before arms registered their own Scope. + scope_key: str | None = None def __post_init__(self) -> None: if type(self.plugin_installed) is not bool or type(self.server_ready) is not bool: @@ -409,7 +433,7 @@ def __post_init__(self) -> None: or self.mcp_requests < 0 ): raise ValueError("Treatment counters must be non-negative integers") - if not self.scope_id: + if not self.scope_id or self.scope_key == "": raise ValueError("Treatment scope must not be empty") def as_dict(self) -> dict[str, object]: @@ -422,6 +446,7 @@ def as_dict(self) -> dict[str, object]: "prompt_sources": self.prompt_sources, "mcp_requests": self.mcp_requests, "scope_id": self.scope_id, + "scope_key": self.scope_key, } @classmethod @@ -440,19 +465,20 @@ def validate_treatment( run_id: str, evidence: TreatmentEvidence, *, + expected_scope_id: str, expected_plugin_version: str, expected_checkout_sha: str, ) -> None: """Fail closed unless the exact expected treatment is proven.""" - expected_scope = f"eval:{run_id}:{arm.value}" common = ( evidence.plugin_installed and evidence.plugin_id == PLUGIN_ID and evidence.plugin_version == expected_plugin_version and evidence.plugin_checkout_sha == expected_checkout_sha and evidence.server_ready - and evidence.scope_id == expected_scope + and evidence.scope_id == expected_scope_id + and evidence.scope_key == scope_key(run_id, arm) ) activity = ( evidence.prompt_sources >= 1 if arm is Arm.ON else evidence.prompt_sources == 0 and evidence.mcp_requests == 0 @@ -1102,6 +1128,7 @@ def _execute_arm( container_started = True self._verify_codex_version(container, paths, store) self._readiness(container, paths, store) + scope_id = self._create_scope(config, arm, container, paths) plugin = self._plugin_list(container, paths) if config.tokensflow_enabled: self._attach_tokensflow_egress(config, container, paths) @@ -1143,7 +1170,7 @@ def _execute_arm( env={ **runtime_proxy_environment(relay_url, config.extra_no_proxy_hosts), "POWERCONTEXT_HOME": "/runtime/pc-home", - "POWERCONTEXT_CODEX_SCOPE_ID": f"eval:{config.run_id}:{arm.value}", + "POWERCONTEXT_CODEX_SCOPE_ID": scope_id, "POWERCONTEXT_EVAL_TRACE_PATH": "/runtime/pc-home/evaluation-injections.jsonl", "UV_PROJECT_ENVIRONMENT": "/runtime/plugin-env", "UV_CACHE_DIR": "/runtime/uv-cache", @@ -1186,13 +1213,14 @@ def _execute_arm( "context/powercontext-injections.jsonl", required=False, ) - evidence = self._evidence(config, arm, container, paths, plugin) + evidence = self._evidence(config, arm, container, paths, plugin, scope_id) if plugin != (PLUGIN_ID, source_provenance.plugin_version): raise InvalidTreatment("Isolated Codex home does not contain the exact expected plugin") validate_treatment( arm, config.run_id, evidence, + expected_scope_id=scope_id, expected_plugin_version=source_provenance.plugin_version, expected_checkout_sha=config.plugin_checkout_sha, ) @@ -2401,7 +2429,6 @@ def _start_container( tokensflow_environment: Mapping[str, str], tokensflow_command_secrets: Sequence[str], ) -> None: - scope = f"eval:{config.run_id}:{arm.value}" tokensflow_mounts: tuple[str, ...] = () tokensflow_container_environment: dict[str, str] = {} executable_path = ( @@ -2469,7 +2496,6 @@ def _start_container( { **runtime_proxy_environment(relay_url, config.extra_no_proxy_hosts), "POWERCONTEXT_HOME": "/runtime/pc-home", - "POWERCONTEXT_CODEX_SCOPE_ID": scope, "UV_PROJECT_ENVIRONMENT": "/runtime/plugin-env", "UV_CACHE_DIR": "/runtime/uv-cache", "UV_PYTHON_INSTALL_DIR": _CONTAINER_UV_PYTHON_INSTALL_DIR, @@ -2565,6 +2591,29 @@ def _readiness(self, container: str, paths: ArmPaths, store: ArtifactStore) -> N ) raise ReadinessFailure(last_reason) + def _create_scope(self, config: SutConfig, arm: Arm, container: str, paths: ArmPaths) -> str: + try: + result = self._docker.run( + ( + "docker", + "exec", + container, + "/runtime/pc-env/bin/python", + "-c", + _SCOPE_CREATION_SCRIPT, + scope_key(config.run_id, arm), + "create-scope", + ), + cwd=paths.runtime, + timeout=30, + ) + scope_id = json.loads(result.stdout)["scope_id"] + if not isinstance(scope_id, str) or not scope_id.strip(): + raise TypeError + except (CommandError, json.JSONDecodeError, KeyError, TypeError) as error: + raise InvalidTreatment("PowerContext Scope could not be created for the arm") from error + return scope_id + def _verify_codex_version(self, container: str, paths: ArmPaths, store: ArtifactStore) -> None: with docker_pressure.heavy_operation(): result = self._docker.run( @@ -2626,8 +2675,8 @@ def _evidence( container: str, paths: ArmPaths, plugin: tuple[str, str], + scope: str, ) -> TreatmentEvidence: - scope = f"eval:{config.run_id}:{arm.value}" query = ( "import json,sqlite3,sys;" "db=sqlite3.connect('/runtime/pc-home/powercontext.db');" @@ -2671,6 +2720,7 @@ def _evidence( prompt_sources=prompt_sources, mcp_requests=mcp_requests, scope_id=scope, + scope_key=scope_key(config.run_id, arm), ) diff --git a/evaluation/src/powercontext_eval/web/models.py b/evaluation/src/powercontext_eval/web/models.py index d960d195cf..cf14ebb18c 100644 --- a/evaluation/src/powercontext_eval/web/models.py +++ b/evaluation/src/powercontext_eval/web/models.py @@ -402,6 +402,7 @@ class TreatmentEvidence(FrozenModel): plugin_installed: bool plugin_version: str scope_id: str + scope_key: str | None = None server_ready: bool diff --git a/evaluation/src/powercontext_eval/web/reporting.py b/evaluation/src/powercontext_eval/web/reporting.py index e462fca355..60899d60ef 100644 --- a/evaluation/src/powercontext_eval/web/reporting.py +++ b/evaluation/src/powercontext_eval/web/reporting.py @@ -30,7 +30,7 @@ from powercontext_eval.artifacts import ArmState from powercontext_eval.benchmarks.swebench_pro.adapter import DATASET_REVISION, HARNESS_COMMIT, SweBenchProInstance -from powercontext_eval.models import Arm, TreatmentMode +from powercontext_eval.models import Arm, TreatmentMode, scope_key from powercontext_eval.report import ArmReport, ReportBundle, TestGroupReport from powercontext_eval.web.baselines import ( BaselineComparison, @@ -249,7 +249,8 @@ def _validate_evidence( and item.plugin_version == configured_plugin_version and item.plugin_installed and item.server_ready - and item.scope_id == f"eval:{run_id}:{arm.value}" + # Evidence recorded before arms registered their own Scope carries the key as its Scope ID. + and (item.scope_key or item.scope_id) == scope_key(run_id, arm) ) activity = ( item.prompt_sources == 0 and item.mcp_requests == 0 diff --git a/evaluation/tests/contract/test_codex_contract.py b/evaluation/tests/contract/test_codex_contract.py index 6956f78183..58fe3f0676 100644 --- a/evaluation/tests/contract/test_codex_contract.py +++ b/evaluation/tests/contract/test_codex_contract.py @@ -326,23 +326,21 @@ def evidence(**changes: object) -> TreatmentEvidence: "server_ready": True, "prompt_sources": 1, "mcp_requests": 1, - "scope_id": "eval:run-1:on", + "scope_id": "scp_on", + "scope_key": "eval:run-1:on", } values.update(changes) return TreatmentEvidence(**values) # type: ignore[arg-type] +OFF_EVIDENCE = {"prompt_sources": 0, "mcp_requests": 0, "scope_id": "scp_off", "scope_key": "eval:run-1:off"} + + def test_treatment_evidence_accepts_valid_on_and_off() -> None: expected = {"expected_plugin_version": "0.1.0", "expected_checkout_sha": "a" * 40} - assert validate_treatment(Arm.ON, "run-1", evidence(), **expected) is None + assert validate_treatment(Arm.ON, "run-1", evidence(), expected_scope_id="scp_on", **expected) is None assert ( - validate_treatment( - Arm.OFF, - "run-1", - evidence(prompt_sources=0, mcp_requests=0, scope_id="eval:run-1:off"), - **expected, - ) - is None + validate_treatment(Arm.OFF, "run-1", evidence(**OFF_EVIDENCE), expected_scope_id="scp_off", **expected) is None ) @@ -355,9 +353,12 @@ def test_treatment_evidence_accepts_valid_on_and_off() -> None: (Arm.ON, {"plugin_checkout_sha": "b" * 40}), (Arm.ON, {"server_ready": False}), (Arm.ON, {"prompt_sources": 0}), - (Arm.ON, {"scope_id": "eval:other:on"}), - (Arm.OFF, {"prompt_sources": 1, "scope_id": "eval:run-1:off"}), - (Arm.OFF, {"mcp_requests": 1, "prompt_sources": 0, "scope_id": "eval:run-1:off"}), + (Arm.ON, {"scope_id": "scp_other"}), + (Arm.ON, {"scope_key": "eval:other:on"}), + # Evidence for a Scope the arm did not register, such as the bare key, is rejected. + (Arm.ON, {"scope_id": "eval:run-1:on", "scope_key": None}), + (Arm.OFF, {**OFF_EVIDENCE, "prompt_sources": 1}), + (Arm.OFF, {**OFF_EVIDENCE, "mcp_requests": 1}), ], ) def test_treatment_evidence_rejects_mismatch(arm: Arm, changes: dict[str, object]) -> None: @@ -366,6 +367,7 @@ def test_treatment_evidence_rejects_mismatch(arm: Arm, changes: dict[str, object arm, "run-1", evidence(**changes), + expected_scope_id=f"scp_{arm.value}", expected_plugin_version="0.1.0", expected_checkout_sha="a" * 40, ) @@ -494,9 +496,12 @@ def run(self, argv: tuple[str, ...], **kwargs: object) -> CommandResult: ) if any(part.endswith("/codex") for part in argv) and "--version" in argv: return command_result("codex-cli 0.145.0\n") + if "create-scope" in argv: + key = argv[argv.index("create-scope") - 1] + return command_result(json.dumps({"scope_id": f"scp_fixture_{key.rsplit(':', 1)[-1]}"})) if "evidence" in argv: scope = argv[argv.index("evidence") - 1] - return command_result(json.dumps({"prompt_sources": 0 if scope.endswith(":off") else 1})) + return command_result(json.dumps({"prompt_sources": 0 if scope.endswith("_off") else 1})) if any(part.endswith("/codex") or part == "codex" for part in argv) and "exec" in argv: result = command_result( '{"type":"agent_message","message":"done"}\n' @@ -524,6 +529,20 @@ def run(self, argv: tuple[str, ...], **kwargs: object) -> CommandResult: return command_result("") +def codex_scope_ids(commands: Sequence[tuple[str, ...]]) -> list[str]: + return [ + part.removeprefix("POWERCONTEXT_CODEX_SCOPE_ID=") + for command in commands + if command[:2] == ("docker", "exec") + for part in command + if part.startswith("POWERCONTEXT_CODEX_SCOPE_ID=") + ] + + +def created_scope_keys(commands: Sequence[tuple[str, ...]]) -> list[str]: + return [command[command.index("create-scope") - 1] for command in commands if "create-scope" in command] + + class FakeRelay: def __init__(self) -> None: self.events: list[tuple[str, str]] = [] @@ -980,7 +999,7 @@ def test_sut_transcript_has_hardening_mount_allowlist_shared_network_and_scope(t assert not any(argument.startswith(f"type=bind,src={Path.home()},") for argument in run) assert f"type=bind,src={paths.tokensflow_home},dst=/root" in joined assert not any(part.startswith(("HOME=", "CODEX_HOME=")) for part in run) - assert "POWERCONTEXT_CODEX_SCOPE_ID=eval:run-1:on" in joined + assert not any(part.startswith("POWERCONTEXT_CODEX_SCOPE_ID=") for part in run) mounts = [run[index + 1] for index, value in enumerate(run) if value in {"--mount", "-v"}] assert all( any( @@ -1022,7 +1041,7 @@ def test_sut_transcript_has_hardening_mount_allowlist_shared_network_and_scope(t assert source_provenance["plugin_version"] == "0.2.0" assert len(source_provenance["plugin_manifest_sha256"]) == 64 evidence_command = next(command for command in transcript if "evidence" in command) - assert "eval:run-1:on" in evidence_command + assert "scp_fixture_on" in evidence_command assert any("pc_sources" in part for part in evidence_command) prewarm_index = next( index @@ -2615,16 +2634,62 @@ def test_distinct_run_ids_derive_distinct_runtime_network_and_scope(tmp_path: Pa ArtifactStore(paths.result_root), ) run = next(command for command in docker.commands if command[:3] == ("docker", "run", "-d")) - evidence_command = next(command for command in docker.commands if "evidence" in command) runtimes.append(paths.runtime) networks.append(run[run.index("--network") + 1]) - scopes.append(evidence_command[evidence_command.index("evidence") - 1]) + scopes.extend(created_scope_keys(docker.commands)) assert runtimes[0] != runtimes[1] assert networks == ["powercontext-eval-parallel-run-a", "powercontext-eval-parallel-run-b"] assert scopes == ["eval:parallel-run-a:on", "eval:parallel-run-b:on"] +def test_codex_runs_in_the_scope_the_server_created_for_the_arm(tmp_path: Path) -> None: + paths = make_paths(tmp_path) + config = sut_config(tmp_path) + config.codex_binary.write_text("binary") + config.uv_binary.write_text("binary") + docker = TranscriptDocker() + + DockerSut(docker, relay_factory=FakeRelay).run_arm( + config, Arm.ON, paths, b"prompt", ArtifactStore(paths.result_root) + ) + + creation = next(index for index, command in enumerate(docker.commands) if "create-scope" in command) + codex_exec = next( + index + for index, command in enumerate(docker.commands) + if any(part.startswith("POWERCONTEXT_CODEX_SCOPE_ID=") for part in command) + ) + assert creation < codex_exec + assert created_scope_keys(docker.commands) == ["eval:run-1:on"] + assert codex_scope_ids(docker.commands) == ["scp_fixture_on"] + treatment = json.loads((paths.result_root / "powercontext/treatment.json").read_text()) + assert treatment["scope_id"] == "scp_fixture_on" + assert treatment["scope_key"] == "eval:run-1:on" + + +def test_arm_fails_closed_before_codex_when_the_scope_cannot_be_created(tmp_path: Path) -> None: + class ScopeCreationFailureDocker(TranscriptDocker): + def run(self, argv: tuple[str, ...], **kwargs: object) -> CommandResult: + if "create-scope" in argv: + self.commands.append(argv) + raise CommandFailed("scope rejected", command_result("", returncode=1)) + return super().run(argv, **kwargs) + + paths = make_paths(tmp_path) + config = sut_config(tmp_path) + config.codex_binary.write_text("binary") + config.uv_binary.write_text("binary") + docker = ScopeCreationFailureDocker() + + with pytest.raises(InvalidTreatment, match="Scope could not be created"): + DockerSut(docker, relay_factory=FakeRelay).run_arm( + config, Arm.ON, paths, b"prompt", ArtifactStore(paths.result_root) + ) + + assert codex_scope_ids(docker.commands) == [] + + def test_sut_uses_timestamp_recorder_and_retains_private_context_traces(tmp_path: Path) -> None: paths = make_paths(tmp_path) config = sut_config(tmp_path) @@ -2698,12 +2763,8 @@ def test_pair_reuses_one_relay_and_network_and_runs_off_then_on(tmp_path: Path) assert sum(command[:3] == ("docker", "network", "rm") for command in docker.commands) == 1 assert relay.events == [("start", "172.29.0.1"), ("stop", "exact")] task_runs = [command for command in docker.commands if command[:3] == ("docker", "run", "-d")] - assert [ - next(value for value in command if value.startswith("POWERCONTEXT_CODEX_SCOPE_ID=")) for command in task_runs - ] == [ - "POWERCONTEXT_CODEX_SCOPE_ID=eval:run-1:off", - "POWERCONTEXT_CODEX_SCOPE_ID=eval:run-1:on", - ] + assert created_scope_keys(docker.commands) == ["eval:run-1:off", "eval:run-1:on"] + assert codex_scope_ids(docker.commands) == ["scp_fixture_off", "scp_fixture_on"] proxy_values = [next(value for value in command if value.startswith("HTTPS_PROXY=")) for command in task_runs] assert proxy_values == ["HTTPS_PROXY=http://172.29.0.1:17890"] * 2 diff --git a/evaluation/tests/web/test_reporting.py b/evaluation/tests/web/test_reporting.py index 0ee1af9219..9a67969cc2 100644 --- a/evaluation/tests/web/test_reporting.py +++ b/evaluation/tests/web/test_reporting.py @@ -206,6 +206,20 @@ def test_rejects_run_outside_root_and_symlink_escape(tmp_path: Path) -> None: load_report(escaped, runs_root) +def test_accepts_evidence_for_the_scopes_each_arm_registered(tmp_path: Path) -> None: + runs_root = tmp_path / "runs" + run_dir = _write_run(runs_root) + for arm in ("off", "on"): + evidence = _evidence(run_dir.name, arm) + evidence.update({"scope_id": f"scp_{arm}", "scope_key": f"eval:{run_dir.name}:{arm}"}) + (run_dir / "arms" / arm / "powercontext" / "treatment.json").write_text(json.dumps(evidence)) + + response = load_report(run_dir, runs_root) + + assert response.evidence.off is not None and response.evidence.off.scope_id == "scp_off" + assert response.evidence.on is not None and response.evidence.on.scope_key == f"eval:{run_dir.name}:on" + + @pytest.mark.parametrize( ("arm", "update"), [ @@ -223,6 +237,8 @@ def test_rejects_run_outside_root_and_symlink_escape(tmp_path: Path) -> None: ("on", {"plugin_installed": False}), ("on", {"server_ready": False}), ("on", {"scope_id": "eval:run-123:off"}), + ("on", {"scope_id": "scp_on", "scope_key": "eval:run-123:off"}), + ("on", {"scope_id": "scp_on", "scope_key": "eval:other:on"}), ], ) def test_rejects_incoherent_treatment_evidence(tmp_path: Path, arm: str, update: dict[str, object]) -> None: From 30d9600607e66f844ed4c7a7364dc50d73bc53be Mon Sep 17 00:00:00 2001 From: Fengzdadi <453788063@qq.com> Date: Wed, 23 Sep 2026 00:54:00 -0400 Subject: [PATCH 2/4] fix(evaluation): reject an empty scope key in report evidence The report loader accepted `scope_key=""` and, because the check used `or`, fell back to `scope_id` as if the evidence predated per-arm Scopes. The harness never writes an empty key, since its own evidence model rejects one, but the report layer re-validates stored evidence independently and should enforce the same rule. Only an absent key now selects the earlier format. Co-Authored-By: Claude Opus 5.5 --- evaluation/src/powercontext_eval/web/models.py | 2 +- evaluation/src/powercontext_eval/web/reporting.py | 2 +- evaluation/tests/web/test_reporting.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/evaluation/src/powercontext_eval/web/models.py b/evaluation/src/powercontext_eval/web/models.py index cf14ebb18c..f1f25c3407 100644 --- a/evaluation/src/powercontext_eval/web/models.py +++ b/evaluation/src/powercontext_eval/web/models.py @@ -402,7 +402,7 @@ class TreatmentEvidence(FrozenModel): plugin_installed: bool plugin_version: str scope_id: str - scope_key: str | None = None + scope_key: Annotated[str, Field(min_length=1)] | None = None server_ready: bool diff --git a/evaluation/src/powercontext_eval/web/reporting.py b/evaluation/src/powercontext_eval/web/reporting.py index 60899d60ef..284046bcbf 100644 --- a/evaluation/src/powercontext_eval/web/reporting.py +++ b/evaluation/src/powercontext_eval/web/reporting.py @@ -250,7 +250,7 @@ def _validate_evidence( and item.plugin_installed and item.server_ready # Evidence recorded before arms registered their own Scope carries the key as its Scope ID. - and (item.scope_key or item.scope_id) == scope_key(run_id, arm) + and (item.scope_key if item.scope_key is not None else item.scope_id) == scope_key(run_id, arm) ) activity = ( item.prompt_sources == 0 and item.mcp_requests == 0 diff --git a/evaluation/tests/web/test_reporting.py b/evaluation/tests/web/test_reporting.py index 9a67969cc2..b254787fba 100644 --- a/evaluation/tests/web/test_reporting.py +++ b/evaluation/tests/web/test_reporting.py @@ -239,6 +239,8 @@ def test_accepts_evidence_for_the_scopes_each_arm_registered(tmp_path: Path) -> ("on", {"scope_id": "eval:run-123:off"}), ("on", {"scope_id": "scp_on", "scope_key": "eval:run-123:off"}), ("on", {"scope_id": "scp_on", "scope_key": "eval:other:on"}), + # An empty key is malformed, not evidence recorded before arms registered their own Scope. + ("on", {"scope_key": ""}), ], ) def test_rejects_incoherent_treatment_evidence(tmp_path: Path, arm: str, update: dict[str, object]) -> None: From f5cb345fe44b5a779a6c338daf00d279887dbbbb Mon Sep 17 00:00:00 2001 From: Fengzdadi <453788063@qq.com> Date: Wed, 23 Sep 2026 01:00:09 -0400 Subject: [PATCH 3/4] fix(evaluation): accept the scope key in console treatment evidence The console parses treatment evidence with a strict schema, and the API now returns `scope_key` for every run: `null` for evidence recorded before arms registered their own Scope, and the run arm key otherwise. Without the field in the schema, the console rejected the evidence of every report, including earlier runs. Co-Authored-By: Claude Opus 5.5 --- evaluation/web/src/api.test.ts | 6 +++++- evaluation/web/src/api.ts | 1 + evaluation/web/src/test/fixtures.ts | 2 ++ evaluation/web/src/types.ts | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/evaluation/web/src/api.test.ts b/evaluation/web/src/api.test.ts index c281c3a45c..153f66ecf4 100644 --- a/evaluation/web/src/api.test.ts +++ b/evaluation/web/src/api.test.ts @@ -418,7 +418,11 @@ describe("EvaluationApi HTTP", () => { elapsed_seconds: { off: 1.5, on: 1, delta: -0.5, percent: -33.333 }, patch_bytes: { off: 0, on: 100, delta: 100, percent: null }, }, - evidence: { off: evidence, on: evidence }, + evidence: { + // Evidence recorded before arms registered their own Scope, and evidence for a registered Scope. + off: { ...evidence, scope_key: null }, + on: { ...evidence, scope_id: "scp_on", scope_key: "eval:task-1:on" }, + }, revisions: { powercontext: "abc" }, configuration: { model: "gpt-5.6-sol" }, generated_at: "2026-07-29T00:02:00Z", diff --git a/evaluation/web/src/api.ts b/evaluation/web/src/api.ts index d59ce8ac2e..064abfeb52 100644 --- a/evaluation/web/src/api.ts +++ b/evaluation/web/src/api.ts @@ -323,6 +323,7 @@ const treatmentEvidenceSchema = z.strictObject({ plugin_installed: z.boolean(), plugin_version: z.string(), scope_id: z.string(), + scope_key: z.string().min(1).nullable(), server_ready: z.boolean(), }); const goldValidationAuditSchema = z.strictObject({ diff --git a/evaluation/web/src/test/fixtures.ts b/evaluation/web/src/test/fixtures.ts index cfa490f436..61dfbbc476 100644 --- a/evaluation/web/src/test/fixtures.ts +++ b/evaluation/web/src/test/fixtures.ts @@ -115,6 +115,7 @@ export const report: ReportResponse = { plugin_installed: true, plugin_version: "0.1.0", scope_id: "eval:run-123:off", + scope_key: null, server_ready: true, }, on: { @@ -125,6 +126,7 @@ export const report: ReportResponse = { plugin_installed: true, plugin_version: "0.1.0", scope_id: "eval:run-123:on", + scope_key: null, server_ready: true, }, }, diff --git a/evaluation/web/src/types.ts b/evaluation/web/src/types.ts index 545dc47851..ac990ee421 100644 --- a/evaluation/web/src/types.ts +++ b/evaluation/web/src/types.ts @@ -233,6 +233,7 @@ export interface TreatmentEvidence { plugin_installed: boolean; plugin_version: string; scope_id: string; + scope_key: string | null; server_ready: boolean; } From 653f92d2de372cca8f59999ddb0da450efc334c4 Mon Sep 17 00:00:00 2001 From: Fengzdadi <453788063@qq.com> Date: Wed, 23 Sep 2026 01:00:10 -0400 Subject: [PATCH 4/4] fix(evaluation): retry an arm whose Scope the Server did not create Scope creation is part of preparing the Server for an arm, like the readiness gate, but its failure was raised as a plain invalid treatment, which the worker treats as terminal. A transient failure now raises a readiness failure with its own reason, so the task is retried with a fresh runtime. Also rename the key helper to `arm_scope_key`, so it no longer reads like the evidence field of the same name. Co-Authored-By: Claude Opus 5.5 --- evaluation/src/powercontext_eval/models.py | 2 +- .../src/powercontext_eval/powercontext_sut.py | 13 ++++++++----- .../src/powercontext_eval/web/reporting.py | 4 ++-- evaluation/tests/contract/test_codex_contract.py | 5 +++-- evaluation/tests/web/test_worker.py | 16 ++++++++++++++++ 5 files changed, 30 insertions(+), 10 deletions(-) diff --git a/evaluation/src/powercontext_eval/models.py b/evaluation/src/powercontext_eval/models.py index e34b3abd0e..ca82844102 100644 --- a/evaluation/src/powercontext_eval/models.py +++ b/evaluation/src/powercontext_eval/models.py @@ -28,7 +28,7 @@ class Arm(StrEnum): ON = "on" -def scope_key(run_id: str, arm: Arm) -> str: +def arm_scope_key(run_id: str, arm: Arm) -> str: """Return the key that identifies the Scope registered for one run arm.""" return f"eval:{run_id}:{arm.value}" diff --git a/evaluation/src/powercontext_eval/powercontext_sut.py b/evaluation/src/powercontext_eval/powercontext_sut.py index 05163886c0..3047045e99 100644 --- a/evaluation/src/powercontext_eval/powercontext_sut.py +++ b/evaluation/src/powercontext_eval/powercontext_sut.py @@ -67,7 +67,7 @@ is_safe_codex_model, ) from powercontext_eval.errors import CommandError, CommandFailed, CommandTimedOut, PowerContextEvalError -from powercontext_eval.models import Arm, scope_key +from powercontext_eval.models import Arm, arm_scope_key from powercontext_eval.process import CommandResult, ProcessRunner from powercontext_eval.tokensflow import ( DrainDeadline, @@ -233,6 +233,7 @@ class ReadinessFailureReason(StrEnum): SERVER_NOT_READY = "server_not_ready" MALFORMED_RESPONSE = "malformed_response" PROBE_FAILED = "probe_failed" + SCOPE_NOT_CREATED = "scope_not_created" _READINESS_FAILURE_SUMMARIES = MappingProxyType( @@ -241,6 +242,7 @@ class ReadinessFailureReason(StrEnum): ReadinessFailureReason.SERVER_NOT_READY: "PowerContext Server remained not ready before the deadline.", ReadinessFailureReason.MALFORMED_RESPONSE: "PowerContext Server returned malformed readiness evidence.", ReadinessFailureReason.PROBE_FAILED: "PowerContext readiness probe failed.", + ReadinessFailureReason.SCOPE_NOT_CREATED: "PowerContext Server did not create the arm Scope.", } ) @@ -478,7 +480,7 @@ def validate_treatment( and evidence.plugin_checkout_sha == expected_checkout_sha and evidence.server_ready and evidence.scope_id == expected_scope_id - and evidence.scope_key == scope_key(run_id, arm) + and evidence.scope_key == arm_scope_key(run_id, arm) ) activity = ( evidence.prompt_sources >= 1 if arm is Arm.ON else evidence.prompt_sources == 0 and evidence.mcp_requests == 0 @@ -2601,7 +2603,7 @@ def _create_scope(self, config: SutConfig, arm: Arm, container: str, paths: ArmP "/runtime/pc-env/bin/python", "-c", _SCOPE_CREATION_SCRIPT, - scope_key(config.run_id, arm), + arm_scope_key(config.run_id, arm), "create-scope", ), cwd=paths.runtime, @@ -2611,7 +2613,8 @@ def _create_scope(self, config: SutConfig, arm: Arm, container: str, paths: ArmP if not isinstance(scope_id, str) or not scope_id.strip(): raise TypeError except (CommandError, json.JSONDecodeError, KeyError, TypeError) as error: - raise InvalidTreatment("PowerContext Scope could not be created for the arm") from error + # Like readiness, a Scope that the Server did not create is a retryable setup failure. + raise ReadinessFailure(ReadinessFailureReason.SCOPE_NOT_CREATED) from error return scope_id def _verify_codex_version(self, container: str, paths: ArmPaths, store: ArtifactStore) -> None: @@ -2720,7 +2723,7 @@ def _evidence( prompt_sources=prompt_sources, mcp_requests=mcp_requests, scope_id=scope, - scope_key=scope_key(config.run_id, arm), + scope_key=arm_scope_key(config.run_id, arm), ) diff --git a/evaluation/src/powercontext_eval/web/reporting.py b/evaluation/src/powercontext_eval/web/reporting.py index 284046bcbf..46de72e64c 100644 --- a/evaluation/src/powercontext_eval/web/reporting.py +++ b/evaluation/src/powercontext_eval/web/reporting.py @@ -30,7 +30,7 @@ from powercontext_eval.artifacts import ArmState from powercontext_eval.benchmarks.swebench_pro.adapter import DATASET_REVISION, HARNESS_COMMIT, SweBenchProInstance -from powercontext_eval.models import Arm, TreatmentMode, scope_key +from powercontext_eval.models import Arm, TreatmentMode, arm_scope_key from powercontext_eval.report import ArmReport, ReportBundle, TestGroupReport from powercontext_eval.web.baselines import ( BaselineComparison, @@ -250,7 +250,7 @@ def _validate_evidence( and item.plugin_installed and item.server_ready # Evidence recorded before arms registered their own Scope carries the key as its Scope ID. - and (item.scope_key if item.scope_key is not None else item.scope_id) == scope_key(run_id, arm) + and (item.scope_key if item.scope_key is not None else item.scope_id) == arm_scope_key(run_id, arm) ) activity = ( item.prompt_sources == 0 and item.mcp_requests == 0 diff --git a/evaluation/tests/contract/test_codex_contract.py b/evaluation/tests/contract/test_codex_contract.py index 58fe3f0676..6b56f275b7 100644 --- a/evaluation/tests/contract/test_codex_contract.py +++ b/evaluation/tests/contract/test_codex_contract.py @@ -2668,7 +2668,7 @@ def test_codex_runs_in_the_scope_the_server_created_for_the_arm(tmp_path: Path) assert treatment["scope_key"] == "eval:run-1:on" -def test_arm_fails_closed_before_codex_when_the_scope_cannot_be_created(tmp_path: Path) -> None: +def test_arm_fails_retryably_before_codex_when_the_scope_cannot_be_created(tmp_path: Path) -> None: class ScopeCreationFailureDocker(TranscriptDocker): def run(self, argv: tuple[str, ...], **kwargs: object) -> CommandResult: if "create-scope" in argv: @@ -2682,11 +2682,12 @@ def run(self, argv: tuple[str, ...], **kwargs: object) -> CommandResult: config.uv_binary.write_text("binary") docker = ScopeCreationFailureDocker() - with pytest.raises(InvalidTreatment, match="Scope could not be created"): + with pytest.raises(ReadinessFailure) as captured: DockerSut(docker, relay_factory=FakeRelay).run_arm( config, Arm.ON, paths, b"prompt", ArtifactStore(paths.result_root) ) + assert captured.value.reason is ReadinessFailureReason.SCOPE_NOT_CREATED assert codex_scope_ids(docker.commands) == [] diff --git a/evaluation/tests/web/test_worker.py b/evaluation/tests/web/test_worker.py index 6119c200f9..e924454c32 100644 --- a/evaluation/tests/web/test_worker.py +++ b/evaluation/tests/web/test_worker.py @@ -1050,6 +1050,22 @@ def runner(config: Any, *, on_phase: Any) -> MinimalRunResult: assert failed.retryable is True +def test_arm_scope_the_server_did_not_create_is_a_retryable_readiness_failure(tmp_path: Path) -> None: + config = _config(tmp_path) + store = _store(config) + task = _create(store) + + def runner(config: Any, *, on_phase: Any) -> MinimalRunResult: + raise ReadinessFailure(ReadinessFailureReason.SCOPE_NOT_CREATED) + + assert EvaluationWorker(config, store, runner=runner, clock=lambda: NOW).run_once() is True + failed = store.get(task.task_id) + assert failed.failure_code is FailureCode.READINESS + assert failed.failure_summary == "PowerContext Server did not create the arm Scope." + assert failed.retry_disposition is RetryDisposition.RETRY + assert failed.retryable is True + + def test_unknown_failure_never_persists_exception_text( tmp_path: Path, caplog: pytest.LogCaptureFixture,