diff --git a/evaluation/src/powercontext_eval/models.py b/evaluation/src/powercontext_eval/models.py index da476af90b..ca82844102 100644 --- a/evaluation/src/powercontext_eval/models.py +++ b/evaluation/src/powercontext_eval/models.py @@ -28,6 +28,12 @@ class Arm(StrEnum): ON = "on" +def arm_scope_key(run_id: str, arm: Arm) -> str: + """Return the key that identifies the Scope registered for one run arm.""" + + return f"eval:{run_id}:{arm.value}" + + class TreatmentMode(StrEnum): """The exact treatment arms executed for one evaluation task.""" diff --git a/evaluation/src/powercontext_eval/powercontext_sut.py b/evaluation/src/powercontext_eval/powercontext_sut.py index 239ea544ac..3047045e99 100644 --- a/evaluation/src/powercontext_eval/powercontext_sut.py +++ b/evaluation/src/powercontext_eval/powercontext_sut.py @@ -67,7 +67,7 @@ is_safe_codex_model, ) from powercontext_eval.errors import CommandError, CommandFailed, CommandTimedOut, PowerContextEvalError -from powercontext_eval.models import Arm +from powercontext_eval.models import Arm, arm_scope_key from powercontext_eval.process import CommandResult, ProcessRunner from powercontext_eval.tokensflow import ( DrainDeadline, @@ -141,6 +141,27 @@ code = 11 sys.exit(code) """.strip() +# The Server generates Scope IDs and rejects an explicit Scope that does not exist, +# so each arm registers its own Scope and hands the returned ID to Codex. +_SCOPE_CREATION_SCRIPT = """ +import json +import sys +from urllib.request import Request, urlopen + +body = { + "title": "Evaluation arm", + "summary": "Isolated Scope for one evaluation arm.", + "idempotency_key": sys.argv[1], +} +request = Request( + "http://127.0.0.1:8000/v1/scopes", + data=json.dumps(body).encode("utf-8"), + headers={"Accept": "application/json", "Content-Type": "application/json"}, + method="POST", +) +with urlopen(request, timeout=10) as response: + print(json.dumps({"scope_id": json.load(response)["scope_id"]})) +""".strip() LOOPBACK_NO_PROXY = "127.0.0.1,localhost,::1" _TOKENSFLOW_RETRY_ATTEMPTS = 10 _PLUGIN_RELATIVE = Path("integrations/codex/plugins/powercontext") @@ -212,6 +233,7 @@ class ReadinessFailureReason(StrEnum): SERVER_NOT_READY = "server_not_ready" MALFORMED_RESPONSE = "malformed_response" PROBE_FAILED = "probe_failed" + SCOPE_NOT_CREATED = "scope_not_created" _READINESS_FAILURE_SUMMARIES = MappingProxyType( @@ -220,6 +242,7 @@ class ReadinessFailureReason(StrEnum): ReadinessFailureReason.SERVER_NOT_READY: "PowerContext Server remained not ready before the deadline.", ReadinessFailureReason.MALFORMED_RESPONSE: "PowerContext Server returned malformed readiness evidence.", ReadinessFailureReason.PROBE_FAILED: "PowerContext readiness probe failed.", + ReadinessFailureReason.SCOPE_NOT_CREATED: "PowerContext Server did not create the arm Scope.", } ) @@ -394,6 +417,9 @@ class TreatmentEvidence: prompt_sources: int mcp_requests: int scope_id: str + # Identifies the run and arm that registered ``scope_id``; absent in evidence + # recorded before arms registered their own Scope. + scope_key: str | None = None def __post_init__(self) -> None: if type(self.plugin_installed) is not bool or type(self.server_ready) is not bool: @@ -409,7 +435,7 @@ def __post_init__(self) -> None: or self.mcp_requests < 0 ): raise ValueError("Treatment counters must be non-negative integers") - if not self.scope_id: + if not self.scope_id or self.scope_key == "": raise ValueError("Treatment scope must not be empty") def as_dict(self) -> dict[str, object]: @@ -422,6 +448,7 @@ def as_dict(self) -> dict[str, object]: "prompt_sources": self.prompt_sources, "mcp_requests": self.mcp_requests, "scope_id": self.scope_id, + "scope_key": self.scope_key, } @classmethod @@ -440,19 +467,20 @@ def validate_treatment( run_id: str, evidence: TreatmentEvidence, *, + expected_scope_id: str, expected_plugin_version: str, expected_checkout_sha: str, ) -> None: """Fail closed unless the exact expected treatment is proven.""" - expected_scope = f"eval:{run_id}:{arm.value}" common = ( evidence.plugin_installed and evidence.plugin_id == PLUGIN_ID and evidence.plugin_version == expected_plugin_version and evidence.plugin_checkout_sha == expected_checkout_sha and evidence.server_ready - and evidence.scope_id == expected_scope + and evidence.scope_id == expected_scope_id + and evidence.scope_key == arm_scope_key(run_id, arm) ) activity = ( evidence.prompt_sources >= 1 if arm is Arm.ON else evidence.prompt_sources == 0 and evidence.mcp_requests == 0 @@ -1102,6 +1130,7 @@ def _execute_arm( container_started = True self._verify_codex_version(container, paths, store) self._readiness(container, paths, store) + scope_id = self._create_scope(config, arm, container, paths) plugin = self._plugin_list(container, paths) if config.tokensflow_enabled: self._attach_tokensflow_egress(config, container, paths) @@ -1143,7 +1172,7 @@ def _execute_arm( env={ **runtime_proxy_environment(relay_url, config.extra_no_proxy_hosts), "POWERCONTEXT_HOME": "/runtime/pc-home", - "POWERCONTEXT_CODEX_SCOPE_ID": f"eval:{config.run_id}:{arm.value}", + "POWERCONTEXT_CODEX_SCOPE_ID": scope_id, "POWERCONTEXT_EVAL_TRACE_PATH": "/runtime/pc-home/evaluation-injections.jsonl", "UV_PROJECT_ENVIRONMENT": "/runtime/plugin-env", "UV_CACHE_DIR": "/runtime/uv-cache", @@ -1186,13 +1215,14 @@ def _execute_arm( "context/powercontext-injections.jsonl", required=False, ) - evidence = self._evidence(config, arm, container, paths, plugin) + evidence = self._evidence(config, arm, container, paths, plugin, scope_id) if plugin != (PLUGIN_ID, source_provenance.plugin_version): raise InvalidTreatment("Isolated Codex home does not contain the exact expected plugin") validate_treatment( arm, config.run_id, evidence, + expected_scope_id=scope_id, expected_plugin_version=source_provenance.plugin_version, expected_checkout_sha=config.plugin_checkout_sha, ) @@ -2401,7 +2431,6 @@ def _start_container( tokensflow_environment: Mapping[str, str], tokensflow_command_secrets: Sequence[str], ) -> None: - scope = f"eval:{config.run_id}:{arm.value}" tokensflow_mounts: tuple[str, ...] = () tokensflow_container_environment: dict[str, str] = {} executable_path = ( @@ -2469,7 +2498,6 @@ def _start_container( { **runtime_proxy_environment(relay_url, config.extra_no_proxy_hosts), "POWERCONTEXT_HOME": "/runtime/pc-home", - "POWERCONTEXT_CODEX_SCOPE_ID": scope, "UV_PROJECT_ENVIRONMENT": "/runtime/plugin-env", "UV_CACHE_DIR": "/runtime/uv-cache", "UV_PYTHON_INSTALL_DIR": _CONTAINER_UV_PYTHON_INSTALL_DIR, @@ -2565,6 +2593,30 @@ def _readiness(self, container: str, paths: ArmPaths, store: ArtifactStore) -> N ) raise ReadinessFailure(last_reason) + def _create_scope(self, config: SutConfig, arm: Arm, container: str, paths: ArmPaths) -> str: + try: + result = self._docker.run( + ( + "docker", + "exec", + container, + "/runtime/pc-env/bin/python", + "-c", + _SCOPE_CREATION_SCRIPT, + arm_scope_key(config.run_id, arm), + "create-scope", + ), + cwd=paths.runtime, + timeout=30, + ) + scope_id = json.loads(result.stdout)["scope_id"] + if not isinstance(scope_id, str) or not scope_id.strip(): + raise TypeError + except (CommandError, json.JSONDecodeError, KeyError, TypeError) as error: + # Like readiness, a Scope that the Server did not create is a retryable setup failure. + raise ReadinessFailure(ReadinessFailureReason.SCOPE_NOT_CREATED) from error + return scope_id + def _verify_codex_version(self, container: str, paths: ArmPaths, store: ArtifactStore) -> None: with docker_pressure.heavy_operation(): result = self._docker.run( @@ -2626,8 +2678,8 @@ def _evidence( container: str, paths: ArmPaths, plugin: tuple[str, str], + scope: str, ) -> TreatmentEvidence: - scope = f"eval:{config.run_id}:{arm.value}" query = ( "import json,sqlite3,sys;" "db=sqlite3.connect('/runtime/pc-home/powercontext.db');" @@ -2671,6 +2723,7 @@ def _evidence( prompt_sources=prompt_sources, mcp_requests=mcp_requests, scope_id=scope, + scope_key=arm_scope_key(config.run_id, arm), ) diff --git a/evaluation/src/powercontext_eval/web/models.py b/evaluation/src/powercontext_eval/web/models.py index d960d195cf..f1f25c3407 100644 --- a/evaluation/src/powercontext_eval/web/models.py +++ b/evaluation/src/powercontext_eval/web/models.py @@ -402,6 +402,7 @@ class TreatmentEvidence(FrozenModel): plugin_installed: bool plugin_version: str scope_id: str + scope_key: Annotated[str, Field(min_length=1)] | None = None server_ready: bool diff --git a/evaluation/src/powercontext_eval/web/reporting.py b/evaluation/src/powercontext_eval/web/reporting.py index e462fca355..46de72e64c 100644 --- a/evaluation/src/powercontext_eval/web/reporting.py +++ b/evaluation/src/powercontext_eval/web/reporting.py @@ -30,7 +30,7 @@ from powercontext_eval.artifacts import ArmState from powercontext_eval.benchmarks.swebench_pro.adapter import DATASET_REVISION, HARNESS_COMMIT, SweBenchProInstance -from powercontext_eval.models import Arm, TreatmentMode +from powercontext_eval.models import Arm, TreatmentMode, arm_scope_key from powercontext_eval.report import ArmReport, ReportBundle, TestGroupReport from powercontext_eval.web.baselines import ( BaselineComparison, @@ -249,7 +249,8 @@ def _validate_evidence( and item.plugin_version == configured_plugin_version and item.plugin_installed and item.server_ready - and item.scope_id == f"eval:{run_id}:{arm.value}" + # Evidence recorded before arms registered their own Scope carries the key as its Scope ID. + and (item.scope_key if item.scope_key is not None else item.scope_id) == arm_scope_key(run_id, arm) ) activity = ( item.prompt_sources == 0 and item.mcp_requests == 0 diff --git a/evaluation/tests/contract/test_codex_contract.py b/evaluation/tests/contract/test_codex_contract.py index 6956f78183..6b56f275b7 100644 --- a/evaluation/tests/contract/test_codex_contract.py +++ b/evaluation/tests/contract/test_codex_contract.py @@ -326,23 +326,21 @@ def evidence(**changes: object) -> TreatmentEvidence: "server_ready": True, "prompt_sources": 1, "mcp_requests": 1, - "scope_id": "eval:run-1:on", + "scope_id": "scp_on", + "scope_key": "eval:run-1:on", } values.update(changes) return TreatmentEvidence(**values) # type: ignore[arg-type] +OFF_EVIDENCE = {"prompt_sources": 0, "mcp_requests": 0, "scope_id": "scp_off", "scope_key": "eval:run-1:off"} + + def test_treatment_evidence_accepts_valid_on_and_off() -> None: expected = {"expected_plugin_version": "0.1.0", "expected_checkout_sha": "a" * 40} - assert validate_treatment(Arm.ON, "run-1", evidence(), **expected) is None + assert validate_treatment(Arm.ON, "run-1", evidence(), expected_scope_id="scp_on", **expected) is None assert ( - validate_treatment( - Arm.OFF, - "run-1", - evidence(prompt_sources=0, mcp_requests=0, scope_id="eval:run-1:off"), - **expected, - ) - is None + validate_treatment(Arm.OFF, "run-1", evidence(**OFF_EVIDENCE), expected_scope_id="scp_off", **expected) is None ) @@ -355,9 +353,12 @@ def test_treatment_evidence_accepts_valid_on_and_off() -> None: (Arm.ON, {"plugin_checkout_sha": "b" * 40}), (Arm.ON, {"server_ready": False}), (Arm.ON, {"prompt_sources": 0}), - (Arm.ON, {"scope_id": "eval:other:on"}), - (Arm.OFF, {"prompt_sources": 1, "scope_id": "eval:run-1:off"}), - (Arm.OFF, {"mcp_requests": 1, "prompt_sources": 0, "scope_id": "eval:run-1:off"}), + (Arm.ON, {"scope_id": "scp_other"}), + (Arm.ON, {"scope_key": "eval:other:on"}), + # Evidence for a Scope the arm did not register, such as the bare key, is rejected. + (Arm.ON, {"scope_id": "eval:run-1:on", "scope_key": None}), + (Arm.OFF, {**OFF_EVIDENCE, "prompt_sources": 1}), + (Arm.OFF, {**OFF_EVIDENCE, "mcp_requests": 1}), ], ) def test_treatment_evidence_rejects_mismatch(arm: Arm, changes: dict[str, object]) -> None: @@ -366,6 +367,7 @@ def test_treatment_evidence_rejects_mismatch(arm: Arm, changes: dict[str, object arm, "run-1", evidence(**changes), + expected_scope_id=f"scp_{arm.value}", expected_plugin_version="0.1.0", expected_checkout_sha="a" * 40, ) @@ -494,9 +496,12 @@ def run(self, argv: tuple[str, ...], **kwargs: object) -> CommandResult: ) if any(part.endswith("/codex") for part in argv) and "--version" in argv: return command_result("codex-cli 0.145.0\n") + if "create-scope" in argv: + key = argv[argv.index("create-scope") - 1] + return command_result(json.dumps({"scope_id": f"scp_fixture_{key.rsplit(':', 1)[-1]}"})) if "evidence" in argv: scope = argv[argv.index("evidence") - 1] - return command_result(json.dumps({"prompt_sources": 0 if scope.endswith(":off") else 1})) + return command_result(json.dumps({"prompt_sources": 0 if scope.endswith("_off") else 1})) if any(part.endswith("/codex") or part == "codex" for part in argv) and "exec" in argv: result = command_result( '{"type":"agent_message","message":"done"}\n' @@ -524,6 +529,20 @@ def run(self, argv: tuple[str, ...], **kwargs: object) -> CommandResult: return command_result("") +def codex_scope_ids(commands: Sequence[tuple[str, ...]]) -> list[str]: + return [ + part.removeprefix("POWERCONTEXT_CODEX_SCOPE_ID=") + for command in commands + if command[:2] == ("docker", "exec") + for part in command + if part.startswith("POWERCONTEXT_CODEX_SCOPE_ID=") + ] + + +def created_scope_keys(commands: Sequence[tuple[str, ...]]) -> list[str]: + return [command[command.index("create-scope") - 1] for command in commands if "create-scope" in command] + + class FakeRelay: def __init__(self) -> None: self.events: list[tuple[str, str]] = [] @@ -980,7 +999,7 @@ def test_sut_transcript_has_hardening_mount_allowlist_shared_network_and_scope(t assert not any(argument.startswith(f"type=bind,src={Path.home()},") for argument in run) assert f"type=bind,src={paths.tokensflow_home},dst=/root" in joined assert not any(part.startswith(("HOME=", "CODEX_HOME=")) for part in run) - assert "POWERCONTEXT_CODEX_SCOPE_ID=eval:run-1:on" in joined + assert not any(part.startswith("POWERCONTEXT_CODEX_SCOPE_ID=") for part in run) mounts = [run[index + 1] for index, value in enumerate(run) if value in {"--mount", "-v"}] assert all( any( @@ -1022,7 +1041,7 @@ def test_sut_transcript_has_hardening_mount_allowlist_shared_network_and_scope(t assert source_provenance["plugin_version"] == "0.2.0" assert len(source_provenance["plugin_manifest_sha256"]) == 64 evidence_command = next(command for command in transcript if "evidence" in command) - assert "eval:run-1:on" in evidence_command + assert "scp_fixture_on" in evidence_command assert any("pc_sources" in part for part in evidence_command) prewarm_index = next( index @@ -2615,16 +2634,63 @@ def test_distinct_run_ids_derive_distinct_runtime_network_and_scope(tmp_path: Pa ArtifactStore(paths.result_root), ) run = next(command for command in docker.commands if command[:3] == ("docker", "run", "-d")) - evidence_command = next(command for command in docker.commands if "evidence" in command) runtimes.append(paths.runtime) networks.append(run[run.index("--network") + 1]) - scopes.append(evidence_command[evidence_command.index("evidence") - 1]) + scopes.extend(created_scope_keys(docker.commands)) assert runtimes[0] != runtimes[1] assert networks == ["powercontext-eval-parallel-run-a", "powercontext-eval-parallel-run-b"] assert scopes == ["eval:parallel-run-a:on", "eval:parallel-run-b:on"] +def test_codex_runs_in_the_scope_the_server_created_for_the_arm(tmp_path: Path) -> None: + paths = make_paths(tmp_path) + config = sut_config(tmp_path) + config.codex_binary.write_text("binary") + config.uv_binary.write_text("binary") + docker = TranscriptDocker() + + DockerSut(docker, relay_factory=FakeRelay).run_arm( + config, Arm.ON, paths, b"prompt", ArtifactStore(paths.result_root) + ) + + creation = next(index for index, command in enumerate(docker.commands) if "create-scope" in command) + codex_exec = next( + index + for index, command in enumerate(docker.commands) + if any(part.startswith("POWERCONTEXT_CODEX_SCOPE_ID=") for part in command) + ) + assert creation < codex_exec + assert created_scope_keys(docker.commands) == ["eval:run-1:on"] + assert codex_scope_ids(docker.commands) == ["scp_fixture_on"] + treatment = json.loads((paths.result_root / "powercontext/treatment.json").read_text()) + assert treatment["scope_id"] == "scp_fixture_on" + assert treatment["scope_key"] == "eval:run-1:on" + + +def test_arm_fails_retryably_before_codex_when_the_scope_cannot_be_created(tmp_path: Path) -> None: + class ScopeCreationFailureDocker(TranscriptDocker): + def run(self, argv: tuple[str, ...], **kwargs: object) -> CommandResult: + if "create-scope" in argv: + self.commands.append(argv) + raise CommandFailed("scope rejected", command_result("", returncode=1)) + return super().run(argv, **kwargs) + + paths = make_paths(tmp_path) + config = sut_config(tmp_path) + config.codex_binary.write_text("binary") + config.uv_binary.write_text("binary") + docker = ScopeCreationFailureDocker() + + with pytest.raises(ReadinessFailure) as captured: + DockerSut(docker, relay_factory=FakeRelay).run_arm( + config, Arm.ON, paths, b"prompt", ArtifactStore(paths.result_root) + ) + + assert captured.value.reason is ReadinessFailureReason.SCOPE_NOT_CREATED + assert codex_scope_ids(docker.commands) == [] + + def test_sut_uses_timestamp_recorder_and_retains_private_context_traces(tmp_path: Path) -> None: paths = make_paths(tmp_path) config = sut_config(tmp_path) @@ -2698,12 +2764,8 @@ def test_pair_reuses_one_relay_and_network_and_runs_off_then_on(tmp_path: Path) assert sum(command[:3] == ("docker", "network", "rm") for command in docker.commands) == 1 assert relay.events == [("start", "172.29.0.1"), ("stop", "exact")] task_runs = [command for command in docker.commands if command[:3] == ("docker", "run", "-d")] - assert [ - next(value for value in command if value.startswith("POWERCONTEXT_CODEX_SCOPE_ID=")) for command in task_runs - ] == [ - "POWERCONTEXT_CODEX_SCOPE_ID=eval:run-1:off", - "POWERCONTEXT_CODEX_SCOPE_ID=eval:run-1:on", - ] + assert created_scope_keys(docker.commands) == ["eval:run-1:off", "eval:run-1:on"] + assert codex_scope_ids(docker.commands) == ["scp_fixture_off", "scp_fixture_on"] proxy_values = [next(value for value in command if value.startswith("HTTPS_PROXY=")) for command in task_runs] assert proxy_values == ["HTTPS_PROXY=http://172.29.0.1:17890"] * 2 diff --git a/evaluation/tests/web/test_reporting.py b/evaluation/tests/web/test_reporting.py index 0ee1af9219..b254787fba 100644 --- a/evaluation/tests/web/test_reporting.py +++ b/evaluation/tests/web/test_reporting.py @@ -206,6 +206,20 @@ def test_rejects_run_outside_root_and_symlink_escape(tmp_path: Path) -> None: load_report(escaped, runs_root) +def test_accepts_evidence_for_the_scopes_each_arm_registered(tmp_path: Path) -> None: + runs_root = tmp_path / "runs" + run_dir = _write_run(runs_root) + for arm in ("off", "on"): + evidence = _evidence(run_dir.name, arm) + evidence.update({"scope_id": f"scp_{arm}", "scope_key": f"eval:{run_dir.name}:{arm}"}) + (run_dir / "arms" / arm / "powercontext" / "treatment.json").write_text(json.dumps(evidence)) + + response = load_report(run_dir, runs_root) + + assert response.evidence.off is not None and response.evidence.off.scope_id == "scp_off" + assert response.evidence.on is not None and response.evidence.on.scope_key == f"eval:{run_dir.name}:on" + + @pytest.mark.parametrize( ("arm", "update"), [ @@ -223,6 +237,10 @@ def test_rejects_run_outside_root_and_symlink_escape(tmp_path: Path) -> None: ("on", {"plugin_installed": False}), ("on", {"server_ready": False}), ("on", {"scope_id": "eval:run-123:off"}), + ("on", {"scope_id": "scp_on", "scope_key": "eval:run-123:off"}), + ("on", {"scope_id": "scp_on", "scope_key": "eval:other:on"}), + # An empty key is malformed, not evidence recorded before arms registered their own Scope. + ("on", {"scope_key": ""}), ], ) def test_rejects_incoherent_treatment_evidence(tmp_path: Path, arm: str, update: dict[str, object]) -> None: diff --git a/evaluation/tests/web/test_worker.py b/evaluation/tests/web/test_worker.py index 6119c200f9..e924454c32 100644 --- a/evaluation/tests/web/test_worker.py +++ b/evaluation/tests/web/test_worker.py @@ -1050,6 +1050,22 @@ def runner(config: Any, *, on_phase: Any) -> MinimalRunResult: assert failed.retryable is True +def test_arm_scope_the_server_did_not_create_is_a_retryable_readiness_failure(tmp_path: Path) -> None: + config = _config(tmp_path) + store = _store(config) + task = _create(store) + + def runner(config: Any, *, on_phase: Any) -> MinimalRunResult: + raise ReadinessFailure(ReadinessFailureReason.SCOPE_NOT_CREATED) + + assert EvaluationWorker(config, store, runner=runner, clock=lambda: NOW).run_once() is True + failed = store.get(task.task_id) + assert failed.failure_code is FailureCode.READINESS + assert failed.failure_summary == "PowerContext Server did not create the arm Scope." + assert failed.retry_disposition is RetryDisposition.RETRY + assert failed.retryable is True + + def test_unknown_failure_never_persists_exception_text( tmp_path: Path, caplog: pytest.LogCaptureFixture, diff --git a/evaluation/web/src/api.test.ts b/evaluation/web/src/api.test.ts index c281c3a45c..153f66ecf4 100644 --- a/evaluation/web/src/api.test.ts +++ b/evaluation/web/src/api.test.ts @@ -418,7 +418,11 @@ describe("EvaluationApi HTTP", () => { elapsed_seconds: { off: 1.5, on: 1, delta: -0.5, percent: -33.333 }, patch_bytes: { off: 0, on: 100, delta: 100, percent: null }, }, - evidence: { off: evidence, on: evidence }, + evidence: { + // Evidence recorded before arms registered their own Scope, and evidence for a registered Scope. + off: { ...evidence, scope_key: null }, + on: { ...evidence, scope_id: "scp_on", scope_key: "eval:task-1:on" }, + }, revisions: { powercontext: "abc" }, configuration: { model: "gpt-5.6-sol" }, generated_at: "2026-07-29T00:02:00Z", diff --git a/evaluation/web/src/api.ts b/evaluation/web/src/api.ts index d59ce8ac2e..064abfeb52 100644 --- a/evaluation/web/src/api.ts +++ b/evaluation/web/src/api.ts @@ -323,6 +323,7 @@ const treatmentEvidenceSchema = z.strictObject({ plugin_installed: z.boolean(), plugin_version: z.string(), scope_id: z.string(), + scope_key: z.string().min(1).nullable(), server_ready: z.boolean(), }); const goldValidationAuditSchema = z.strictObject({ diff --git a/evaluation/web/src/test/fixtures.ts b/evaluation/web/src/test/fixtures.ts index cfa490f436..61dfbbc476 100644 --- a/evaluation/web/src/test/fixtures.ts +++ b/evaluation/web/src/test/fixtures.ts @@ -115,6 +115,7 @@ export const report: ReportResponse = { plugin_installed: true, plugin_version: "0.1.0", scope_id: "eval:run-123:off", + scope_key: null, server_ready: true, }, on: { @@ -125,6 +126,7 @@ export const report: ReportResponse = { plugin_installed: true, plugin_version: "0.1.0", scope_id: "eval:run-123:on", + scope_key: null, server_ready: true, }, }, diff --git a/evaluation/web/src/types.ts b/evaluation/web/src/types.ts index 545dc47851..ac990ee421 100644 --- a/evaluation/web/src/types.ts +++ b/evaluation/web/src/types.ts @@ -233,6 +233,7 @@ export interface TreatmentEvidence { plugin_installed: boolean; plugin_version: string; scope_id: string; + scope_key: string | null; server_ready: boolean; }