From f25fb15e4664a7afab371f696eae71dedd8876fe Mon Sep 17 00:00:00 2001 From: KeyffMS <124252104+KeyffMS@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:51:23 +0200 Subject: [PATCH 1/5] Reproduce C09 timeout, C10 hook-source and C13 process-contract defects --- ...test_qualification_execution_boundaries.py | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 tests/test_qualification_execution_boundaries.py diff --git a/tests/test_qualification_execution_boundaries.py b/tests/test_qualification_execution_boundaries.py new file mode 100644 index 0000000..57bfa7d --- /dev/null +++ b/tests/test_qualification_execution_boundaries.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +from contextlib import ExitStack, nullcontext +import json +import os +from pathlib import Path +import re +import shlex +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) + +import live_codex_qualification_c10 as c10 +import live_codex_qualification_harness_v4 as v4 +import live_codex_qualification_harness_v7 as v7 + +base = v4.base + + +def execute_hook(command: str, cwd: Path, event: dict) -> subprocess.CompletedProcess[str]: + # Execute the actual generated POSIX command on the live runner's platform. + # Windows exercises the same generated Python program and argument contract. + if os.name == "nt": + argv = shlex.split(command.replace("$(git rev-parse --show-toplevel)", str(cwd))) + argv[0] = sys.executable + else: + argv = ["/bin/sh", "-c", command] + return subprocess.run( + argv, cwd=cwd, input=json.dumps(event), text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=30, check=False, + ) + + +class SubagentProcessContractTests(unittest.TestCase): + def test_active_c13_configuration_executes_its_generated_proxy(self) -> None: + with tempfile.TemporaryDirectory(prefix="plananvil c13 ") as tmp: + root = Path(tmp) + repo = root / "project with spaces" + proof = "c" * 32 + captured = {} + + def probe(**_kwargs): + base.ensure_git_repo(repo) + v7._seed_declared_project_fixture(repo, proof) + base.commit_fixture_baseline(repo) + command = base.load_json(repo / ".codex/hooks.json")["hooks"]["SubagentStart"][0]["hooks"][0]["command"] + log = v7.prior._hook_log(repo) + with v7.compat.regression._hook_env(log): + before = base.git_snapshot(repo) + completed = execute_hook(command, repo, { + "hook_event_name": "SubagentStart", "cwd": str(repo), + "agent_type": "fixture_agent", "session_id": "offline-test", + }) + after = base.git_snapshot(repo) + captured.update(completed=completed, before=before, after=after, + records=v7.prior._read_hook_records(repo)) + return "OFFLINE_ONLY", False + + # Same compatibility context and project fixture used by v7 live C13. + v7.compat.run_c13(probe, runtime_root=root) + completed = captured["completed"] + self.assertEqual(completed.returncode, 0, completed.stderr) + output = json.loads(completed.stdout) + self.assertIs(output["continue"], False) + self.assertEqual(output["hookSpecificOutput"]["hookEventName"], "SubagentStart") + self.assertEqual(output["hookSpecificOutput"]["additionalContext"], "C13_CONTEXT_TOKEN=" + proof) + self.assertEqual(len(captured["records"]), 1) + self.assertEqual(captured["records"][0]["event"], "SubagentStart") + self.assertEqual(captured["records"][0]["returncode"], 0) + self.assertEqual(captured["before"], captured["after"]) + + +class CompactionCompletionTests(unittest.TestCase): + def evaluate(self, payload, events, error, records=None): + with tempfile.TemporaryDirectory() as tmp, ExitStack() as stack: + root = Path(tmp) + if records is None: + records = [ + {"event": "PreCompact"}, {"event": "PostCompact"}, + {"event": "PreCompact"}, {"event": "PostCompact"}, + {"event": "PreToolUse"}, + ] + stack.enter_context(mock.patch.object(v4, "_runtime_paths", return_value=(root,) * 7)) + stack.enter_context(mock.patch.object(base, "ensure_git_repo")) + stack.enter_context(mock.patch.object(base, "git", return_value="a" * 40)) + stack.enter_context(mock.patch.object(base, "git_snapshot", return_value={"head": "a" * 40})) + stack.enter_context(mock.patch.object(v4, "_start_active_run", return_value=(root, ".pursue/runs/test"))) + stack.enter_context(mock.patch.object(v4, "_checkpoint_validation", return_value={"ok": True})) + stack.enter_context(mock.patch.object(v4, "_clear_hook_log")) + stack.enter_context(mock.patch.object(v4, "_read_hook_records", return_value=records)) + stack.enter_context(mock.patch.object(v4, "_run_codex_probe", return_value=(payload, events, error))) + writer = stack.enter_context(mock.patch.object(v4, "_write_result", side_effect=lambda **kw: (kw["result"], True))) + v4._c09_runtime(root=root, runtime_root=root, schemas={}, version="offline", + os_name="test", source_commit="b" * 40, date="2026-09-05") + return writer.call_args.kwargs + + def test_timeout_cannot_pass_even_after_two_compactions(self) -> None: + result = self.evaluate({}, {"timeout": True}, "Codex invocation timed out") + self.assertEqual(result["result"], "BLOCKED") + self.assertFalse(result["expected_met"]) + self.assertNotEqual(result["trials"][0]["outcome"], "PASS") + + def test_missing_completion_payload_cannot_pass(self) -> None: + result = self.evaluate({}, {}, None) + self.assertEqual(result["result"], "BLOCKED") + + def test_completed_positive_trial_still_passes(self) -> None: + result = self.evaluate({"capability_id": "C09", "outcome": "PASS"}, {}, None) + self.assertEqual(result["result"], "REPRODUCED") + self.assertTrue(result["expected_met"]) + + def test_second_compaction_still_requires_subsequent_tool_use(self) -> None: + result = self.evaluate({"capability_id": "C09", "outcome": "PASS"}, {}, None, + [{"event": "PreCompact"}, {"event": "PostCompact"}] * 2) + self.assertNotEqual(result["result"], "REPRODUCED") + + def test_observed_stop_with_valid_checkpoint_still_fails(self) -> None: + result = self.evaluate({"capability_id": "C09", "outcome": "PASS"}, {}, None, + [{"event": "PreCompact", "continue": False}, + {"event": "PostCompact"}] * 2 + [{"event": "PreToolUse"}]) + self.assertEqual(result["result"], "FAILED") + + +class RecoveryFixtureExecutionTests(unittest.TestCase): + def test_both_recovery_trials_use_independent_correctly_scoped_fixtures(self) -> None: + """Offline lifecycle driver, NOT evidence of a live Codex capability. + + Only the Codex process is replaced. Git, the release installer, start, + checkpoint creator/validator, generated commands and product hooks run. + The driver deliberately reads declarations from the root checkout, as + rust-v0.153.4 config/src/loader/mod.rs does for linked worktrees. + """ + with tempfile.TemporaryDirectory(prefix="plananvil c10 ") as tmp: + runtime = Path(tmp) + schemas = base.write_schemas(runtime / "schemas") + actual_run = base.run + roots = [] + configured_events = [] + proofs = [] + + def driver(args, *, cwd, **kwargs): + if args[0] != "codex": + return actual_run(args, cwd=cwd, **kwargs) + self.assertEqual(len(roots) < 2, True, "Unexpected extra Codex invocation") + self.assertEqual(args[args.index("--sandbox") + 1], "read-only") + common_dir = Path(actual_run( + ["git", "rev-parse", "--path-format=absolute", "--git-common-dir"], + cwd=cwd, timeout=30, + ).stdout.strip()) + source = common_dir.parent + roots.append(source) + hooks = base.load_json(source / ".codex/hooks.json")["hooks"] + configured_events.append(set(hooks)) + postcompact = len(roots) == 2 + lifecycle = [("SessionStart", "startup")] + if postcompact: + lifecycle += [("PreCompact", "auto"), ("PostCompact", "auto"), + ("SessionStart", "compact")] + contexts = [] + for event_name, trigger in lifecycle: + for group in hooks.get(event_name, []): + if group.get("matcher") and not re.search(group["matcher"], trigger): + continue + for handler in group["hooks"]: + event = {"hook_event_name": event_name, "cwd": str(cwd), + "source": trigger, "trigger": trigger, "session_id": "offline-test"} + completed = execute_hook(handler["command"], cwd, event) + self.assertEqual(completed.returncode, 0, completed.stderr) + parsed = json.loads(completed.stdout) if completed.stdout.strip() else {} + self.assertIsNot(parsed.get("continue"), False, parsed) + context = parsed.get("hookSpecificOutput", {}).get("additionalContext") + if context: + contexts.append(context) + matches = re.findall(r"evidence/c10-recovery-([0-9a-f]{32})\.json", "\n".join(contexts)) + self.assertTrue(matches, "The real product hook did not return the fixture recovery pointer") + proof = matches[-1] + proofs.append(proof) + payload = { + "capability_id": "C10", + "trial": "postcompact_recovery_context" if postcompact else "session_start_recovery_context", + "outcome": "PASS", "assertions": [], "blocker": None, + "observations": ["C10_RECOVERY_ECHO=" + proof], + } + base.json_dump(Path(args[args.index("-o") + 1]), payload) + events = [{"type": "turn.started"}] + if postcompact: + events.append({"type": "item.completed", "item": { + "id": "offline-command", "type": "command_execution", + "command": "cat qualification-payload/segment-01.txt", + "status": "completed", "exit_code": 0, "aggregated_output": "fixture payload", + }}) + events.append({"type": "turn.completed"}) + return subprocess.CompletedProcess(args, 0, "\n".join(map(json.dumps, events)), "") + + with mock.patch.object(base, "run", side_effect=driver), mock.patch.object( + v4, "_write_result", side_effect=lambda **kw: (kw["result"], True) + ) as writer: + result, _required = c10.run_c10( + root=ROOT, runtime_root=runtime, schemas=schemas, version="codex-cli 0.153.4", + os_name="offline-test", source_commit="b" * 40, date="2026-09-05", + live_trust_runtime=lambda: nullcontext(runtime), + ) + evidence = writer.call_args.kwargs + self.assertEqual(result, "REPRODUCED", evidence) + self.assertEqual(len(roots), 2) + self.assertNotEqual(roots[0], roots[1], "A second invocation must not reuse the first source checkout") + self.assertIn("SessionStart", configured_events[0]) + self.assertNotIn("SessionStart", configured_events[1]) + self.assertIn("PreCompact", configured_events[1]) + self.assertIn("PostCompact", configured_events[1]) + self.assertNotEqual(proofs[0], proofs[1], "Recovery probes must have independent proof values") + for proof in proofs: + self.assertNotIn(proof, json.dumps(evidence), "Opaque proof leaked into persisted evidence") + + +if __name__ == "__main__": + unittest.main() From 2bd212f6110339dba5c44fe86d99e9a9f02f51f2 Mon Sep 17 00:00:00 2001 From: KeyffMS <124252104+KeyffMS@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:53:21 +0200 Subject: [PATCH 2/5] Pass the event and script required by the active C13 hook proxy --- tools/live_codex_qualification_harness_v6.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/live_codex_qualification_harness_v6.py b/tools/live_codex_qualification_harness_v6.py index 7babd56..6737342 100644 --- a/tools/live_codex_qualification_harness_v6.py +++ b/tools/live_codex_qualification_harness_v6.py @@ -79,6 +79,7 @@ def _seed_project_fixture(repo: Path, context_proof: str, *, include_project_age "command": ( 'python3 "$(git rev-parse --show-toplevel)/.codex/hooks/' 'qualification-c13-hook-proxy.py"' + ' SubagentStart subagent-start-fixture.py' ), "timeout": 30, } From 5a7c9b487381b566dd6d744194bb5ae232d578e8 Mon Sep 17 00:00:00 2001 From: KeyffMS <124252104+KeyffMS@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:56:16 +0200 Subject: [PATCH 3/5] Prepare hook sources before bootstrap and require completed C09 evidence --- tools/live_codex_qualification_harness_v4.py | 39 +++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/tools/live_codex_qualification_harness_v4.py b/tools/live_codex_qualification_harness_v4.py index dc51ab0..01c551a 100644 --- a/tools/live_codex_qualification_harness_v4.py +++ b/tools/live_codex_qualification_harness_v4.py @@ -7,7 +7,7 @@ import subprocess import sys from pathlib import Path -from typing import Any +from typing import Any, Callable import live_codex_qualification_harness as v1 import live_codex_qualification_harness_v2 as v2 @@ -449,6 +449,7 @@ def _start_active_run( create_checkpoint: bool, segments: int, segment_bytes: int, + prepare_repo: Callable[[Path], None] | None = None, ) -> tuple[Path, str]: v1._install_plananvil_release(root, repo) _instrument_hooks( @@ -470,6 +471,10 @@ def _start_active_run( repeats = max(1, segment_bytes // len(marker)) text = (marker * repeats)[:segment_bytes] _write(payload_dir / f"segment-{index:02d}.txt", text + "\n") + # Configure the actual root-checkout hook source before any source snapshot, + # linked worktree, active run or checkpoint exists. Defaults are unchanged. + if prepare_repo is not None: + prepare_repo(repo) base.git(repo, "add", "-A") base.git(repo, "commit", "--allow-empty", "-q", "-m", "Install deterministic compaction fixture") @@ -841,6 +846,15 @@ def _c09_runtime( continued_after_second = _continued_after_second_postcompact(records) checkpoint_coherent = bool(checkpoint_before.get("ok")) and bool(checkpoint_after.get("ok")) no_stop_loop = not stops and continued_after_second + invocation_completed = ( + error is None + and not events.get("timeout") + and payload.get("capability_id") == capability_id + and payload.get("outcome") == "PASS" + ) + completion_blocker = error or ( + None if invocation_completed else "C09 did not return a completed positive structured result." + ) trial = { "capability_id": capability_id, @@ -848,10 +862,15 @@ def _c09_runtime( "trial_name": "checkpoint_auto_compact_recover_recompact", "outcome": ( "BLOCKED" - if not two_compactions + if not invocation_completed or not two_compactions else ("PASS" if checkpoint_coherent and no_stop_loop else "FAIL") ), "assertions": [ + { + "name": "codex_invocation_completed_without_timeout", + "status": "PASS" if invocation_completed else "BLOCKED", + "evidence": f"invocation_completed={str(invocation_completed).lower()}", + }, { "name": "valid_checkpoint_allows_compaction", "status": "PASS" if two_compactions and not stops else ("BLOCKED" if not two_compactions else "FAIL"), @@ -867,10 +886,11 @@ def _c09_runtime( }, { "name": "second_valid_compaction_path_is_not_permanently_blocked", - "status": "PASS" if two_compactions and no_stop_loop else ("BLOCKED" if not two_compactions else "FAIL"), + "status": "PASS" if two_compactions and no_stop_loop and invocation_completed else ("BLOCKED" if not two_compactions or not invocation_completed else "FAIL"), "evidence": ( f"second_postcompact_observed={str(len(post) >= 2).lower()}; " - f"tool_use_after_second_postcompact={str(continued_after_second).lower()}" + f"tool_use_after_second_postcompact={str(continued_after_second).lower()}; " + f"invocation_completed={str(invocation_completed).lower()}" ), }, ], @@ -882,8 +902,9 @@ def _c09_runtime( f"checkpoint_before_valid={str(bool(checkpoint_before.get('ok'))).lower()}", f"checkpoint_after_valid={str(bool(checkpoint_after.get('ok'))).lower()}", f"invocation_error={error or 'none'}", + f"invocation_completed={str(invocation_completed).lower()}", ], - "blocker": error if not two_compactions else None, + "blocker": completion_blocker, "event_summary": events, "git_before": before, "git_after": after, @@ -902,6 +923,11 @@ def _c09_runtime( expected_met = False blocker = "The deterministic C09 fixture did not begin with a valid checkpoint." summary = "C09 blocked during deterministic fixture preparation." + elif not invocation_completed: + result = "BLOCKED" + expected_met = False + blocker = completion_blocker + summary = "C09 blocked because partial lifecycle observations do not prove successful completion." elif not two_compactions: result = "BLOCKED" expected_met = False @@ -939,6 +965,7 @@ def _c09_runtime( f"postcompact_count={len(post)}", f"continued_after_second={str(continued_after_second).lower()}", f"checkpoint_after_valid={str(bool(checkpoint_after.get('ok'))).lower()}", + f"invocation_completed={str(invocation_completed).lower()}", ], blocker=blocker, summary=summary, @@ -969,4 +996,4 @@ def main(argv: list[str] | None = None) -> int: if __name__ == "__main__": - raise SystemExit(main()) + raise SystemExit(main()) \ No newline at end of file From af1af0e654c7eace64e954dc87b8fb2da08dded7 Mon Sep 17 00:00:00 2001 From: KeyffMS <124252104+KeyffMS@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:58:28 +0200 Subject: [PATCH 4/5] Isolate C10 at the real root-checkout hook source before bootstrap --- tools/live_codex_qualification_c10.py | 113 +++++++++++++++++++++----- 1 file changed, 91 insertions(+), 22 deletions(-) diff --git a/tools/live_codex_qualification_c10.py b/tools/live_codex_qualification_c10.py index 6d6cafd..ba8fce1 100644 --- a/tools/live_codex_qualification_c10.py +++ b/tools/live_codex_qualification_c10.py @@ -47,15 +47,15 @@ def _inject_recovery_probe_state(planning: Path, run_root: str, proof: str) -> N base.json_dump(state_path, state) -def _set_compaction_trigger(planning: Path) -> None: - """Make genuine auto-compaction deterministic without changing product defaults.""" +def _set_compaction_trigger(repo: Path) -> None: + """Prepare the disposable root checkout before the product captures its state.""" v4._set_compact_config( - planning, + repo, limit=C10_COMPACT_LIMIT, scope=v4.COMPACT_SCOPE, ) - config_path = planning / ".codex" / "config.toml" + config_path = repo / ".codex" / "config.toml" text = config_path.read_text(encoding="utf-8") if config_path.exists() else "" config_path.write_text( compat._set_feature(text, "token_budget", "false"), @@ -63,10 +63,10 @@ def _set_compaction_trigger(planning: Path) -> None: ) -def _disable_session_start_for_postcompact(planning: Path) -> bool: - """Isolate PostCompact so the opaque proof cannot arrive through SessionStart.""" +def _disable_session_start_for_postcompact(repo: Path) -> bool: + """Modify the root-checkout hook source, not a linked-worktree override.""" - hooks_path = planning / ".codex" / "hooks.json" + hooks_path = repo / ".codex" / "hooks.json" hooks = base.load_json(hooks_path) configured = hooks.get("hooks") if isinstance(hooks, dict) else None if not isinstance(configured, dict): @@ -80,6 +80,38 @@ def _disable_session_start_for_postcompact(planning: Path) -> bool: ) +def _prepare_postcompact_repo(repo: Path) -> None: + # Codex rust-v0.153.4 loads linked-worktree hook declarations from the + # corresponding root checkout. This runs BEFORE commit/start/checkpoint. + if not _disable_session_start_for_postcompact(repo): + raise base.QualificationError("C10 could not isolate the root-checkout hook source") + _set_compaction_trigger(repo) + + +def _postcompact_hooks_isolated(repo: Path, planning: Path) -> bool: + source = repo / ".codex" / "hooks.json" + local = planning / ".codex" / "hooks.json" + hooks = base.load_json(source).get("hooks", {}) + return ( + source.read_bytes() == local.read_bytes() + and "SessionStart" not in hooks + and bool(hooks.get("PreCompact")) + and bool(hooks.get("PostCompact")) + ) + + +def _redact_proofs(value: Any, proofs: tuple[str, ...]) -> Any: + if isinstance(value, str): + for proof in proofs: + value = value.replace(proof, "") + return value + if isinstance(value, list): + return [_redact_proofs(item, proofs) for item in value] + if isinstance(value, dict): + return {key: _redact_proofs(item, proofs) for key, item in value.items()} + return value + + def _session_prompt() -> str: return f"""Exercise the real C10 SessionStart recovery-context path. @@ -151,6 +183,7 @@ def run_c10( ) = v4._runtime_paths(root=root, runtime_root=runtime_root, capability_id=capability_id) proof = secrets.token_hex(16) + compact_proof = secrets.token_hex(16) setup_error: str | None = None session_error: str | None = None compact_error: str | None = None @@ -169,7 +202,10 @@ def run_c10( checkpoint_before_compact: dict[str, Any] = {"ok": False} checkpoint_after_compact: dict[str, Any] = {"ok": False} postcompact_isolated = False + source_session_unchanged = False + source_compact_unchanged = False fixture_commit = "unavailable" + compact_fixture_commit = "unavailable" try: with v2._python_bytecode_disabled(): @@ -195,6 +231,7 @@ def run_c10( with live_trust_runtime(): v4._clear_hook_log(planning) + source_before_session = base.git_snapshot(repo) before_session = base.git_snapshot(planning) session_payload, session_events, session_error = v4._run_codex_probe( cwd=planning, @@ -206,16 +243,41 @@ def run_c10( timeout=600, ) after_session = base.git_snapshot(planning) + source_session_unchanged = source_before_session == base.git_snapshot(repo) session_records = v4._read_hook_records(planning) checkpoint_after_session = v4._checkpoint_validation(planning) - postcompact_isolated = _disable_session_start_for_postcompact(planning) - _set_compaction_trigger(planning) - checkpoint_before_compact = v4._checkpoint_validation(planning) - v4._clear_hook_log(planning) - before_compact = base.git_snapshot(planning) + # Independent source checkout and proof: no edits to the first + # source after bootstrap, and no SessionStart proof carry-over. + compact_repo = cap_runtime / "postcompact-repo" + compact_worktrees = worktrees / "postcompact" + compact_worktrees.mkdir(parents=True, exist_ok=True) + base.ensure_git_repo(compact_repo) + compact_planning, compact_run_root = v4._start_active_run( + root=root, + repo=compact_repo, + worktrees=compact_worktrees, + version=version, + compact_limit=C10_COMPACT_LIMIT, + create_checkpoint=False, + segments=1, + segment_bytes=C10_SEGMENT_BYTES, + prepare_repo=_prepare_postcompact_repo, + ) + compact_fixture_commit = base.git(compact_repo, "rev-parse", "HEAD") + postcompact_isolated = _postcompact_hooks_isolated(compact_repo, compact_planning) + if not postcompact_isolated: + raise base.QualificationError("C10 root-checkout PostCompact isolation failed") + _inject_recovery_probe_state(compact_planning, compact_run_root, compact_proof) + v4._create_checkpoint(planning=compact_planning, run_root=compact_run_root) + checkpoint_before_compact = v4._checkpoint_validation(compact_planning) + if not bool(checkpoint_before_compact.get("ok")): + raise base.QualificationError("C10 PostCompact fixture checkpoint is invalid") + v4._clear_hook_log(compact_planning) + source_before_compact = base.git_snapshot(compact_repo) + before_compact = base.git_snapshot(compact_planning) compact_payload, compact_events, compact_error = v4._run_codex_probe( - cwd=planning, + cwd=compact_planning, prompt=_postcompact_prompt(), schemas=schemas, results_dir=results_dir, @@ -225,9 +287,10 @@ def run_c10( compact_scope=v4.COMPACT_SCOPE, timeout=900, ) - after_compact = base.git_snapshot(planning) - compact_records = v4._read_hook_records(planning) - checkpoint_after_compact = v4._checkpoint_validation(planning) + after_compact = base.git_snapshot(compact_planning) + source_compact_unchanged = source_before_compact == base.git_snapshot(compact_repo) + compact_records = v4._read_hook_records(compact_planning) + checkpoint_after_compact = v4._checkpoint_validation(compact_planning) except Exception as exc: setup_error = base.sanitize_text(f"{type(exc).__name__}: {exc}") @@ -235,7 +298,7 @@ def run_c10( session_context = [item for item in session_start if item.get("additional_context")] session_echo = _exact_echo(session_payload, proof) session_no_tools = int(session_events.get("completed_command_items") or 0) == 0 - session_unchanged = bool(before_session) and before_session == after_session + session_unchanged = bool(before_session) and before_session == after_session and source_session_unchanged session_checkpoint_ok = ( bool(checkpoint_before.get("ok")) and bool(checkpoint_after_session.get("ok")) ) @@ -255,9 +318,9 @@ def run_c10( compact_session_start = v4._event_records(compact_records, "SessionStart") post_context = [item for item in postcompact if item.get("additional_context")] compact_stops = [item for item in precompact if item.get("continue") is False] - compact_echo = _exact_echo(compact_payload, proof) + compact_echo = _exact_echo(compact_payload, compact_proof) compact_one_command = int(compact_events.get("completed_command_items") or 0) == 1 - compact_unchanged = bool(before_compact) and before_compact == after_compact + compact_unchanged = bool(before_compact) and before_compact == after_compact and source_compact_unchanged compact_checkpoint_ok = ( bool(checkpoint_before_compact.get("ok")) and bool(checkpoint_after_compact.get("ok")) @@ -318,6 +381,7 @@ def run_c10( f"command_items={int(session_events.get('completed_command_items') or 0)}", f"checkpoint_valid={str(session_checkpoint_ok).lower()}", f"repository_unchanged={str(session_unchanged).lower()}", + f"source_worktree_unchanged={str(source_session_unchanged).lower()}", ], "blocker": session_error or setup_error, "event_summary": session_events, @@ -385,17 +449,22 @@ def run_c10( f"command_items={int(compact_events.get('completed_command_items') or 0)}", f"checkpoint_valid={str(compact_checkpoint_ok).lower()}", f"repository_unchanged={str(compact_unchanged).lower()}", + f"source_worktree_unchanged={str(source_compact_unchanged).lower()}", ], "blocker": compact_error or setup_error, "event_summary": compact_events, "checkpoint_before": checkpoint_before_compact, "checkpoint_after": checkpoint_after_compact, "model_payload_summary": _payload_summary(compact_payload), + "fixture_commit": compact_fixture_commit, "config_evidence": { "model_auto_compact_token_limit": C10_COMPACT_LIMIT, "model_auto_compact_token_limit_scope": v4.COMPACT_SCOPE, "token_budget_disabled_in_isolated_fixture": True, "session_start_removed_only_for_postcompact_isolation": True, + "hook_source": "disposable_root_checkout", + "configured_before_bootstrap": True, + "independent_recovery_proof": True, }, } @@ -430,13 +499,13 @@ def run_c10( f"postcompact_recovery_context={str(compact_ok).lower()}", "opaque_recovery_value_persisted=false", ], - blocker=blocker, + blocker=_redact_proofs(blocker, (proof, compact_proof)), summary=( - "C10 reproduced with an outer-harness-created active PlanAnvil run and product-validated checkpoint; real SessionStart and isolated real PostCompact each supplied recovery context to the model." + "C10 reproduced with independent outer-harness-created PlanAnvil runs and product-validated checkpoints; real SessionStart and isolated real PostCompact each supplied recovery context to the model." if met else "C10 deterministic recovery qualification did not completely reproduce both lifecycle context paths." ), - trials=[session_trial, compact_trial], + trials=_redact_proofs([session_trial, compact_trial], (proof, compact_proof)), fixture_commit=fixture_commit, version=version, os_name=os_name, From ef76142edb4d435e87c5a3010f223c7cf05b22da Mon Sep 17 00:00:00 2001 From: KeyffMS <124252104+KeyffMS@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:11:32 +0200 Subject: [PATCH 5/5] Add cross-platform executable verification and a scoped recovery handoff Keep C09/C10/C13 expected assertions and full release requirements unchanged. Align materialized configuration docs with the pinned loader and v7 project-scoped transport; run the same live runtimes in recovery mode without claiming a full release pass. --- .../plananvil-codex-qualification.yml | 29 +++-- .github/workflows/plananvil-tests.yml | 2 + ...UALIFICATION_EXECUTION_AUDIT_2026-09-05.md | 58 +++++++++ tests/test_live_codex_qualification_c10.py | 41 +++++-- tests/test_prepare_capabilities_overlay.py | 28 ++++- ...test_qualification_execution_boundaries.py | 2 +- tests/test_qualification_recovery.py | 100 +++++++++++++++ tools/live_codex_qualification_recovery.py | 102 ++++++++++++++++ tools/prepare_capabilities.py | 115 +++++++++++------- 9 files changed, 408 insertions(+), 69 deletions(-) create mode 100644 docs/CODEX_QUALIFICATION_EXECUTION_AUDIT_2026-09-05.md create mode 100644 tests/test_qualification_recovery.py create mode 100644 tools/live_codex_qualification_recovery.py diff --git a/.github/workflows/plananvil-codex-qualification.yml b/.github/workflows/plananvil-codex-qualification.yml index d9f4d0b..3826eb2 100644 --- a/.github/workflows/plananvil-codex-qualification.yml +++ b/.github/workflows/plananvil-codex-qualification.yml @@ -13,6 +13,7 @@ on: - c13 - diagnostics - precision + - recovery - full permissions: @@ -247,7 +248,7 @@ jobs: if: >- github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && - (inputs.mode == 'full' || inputs.mode == 'c13') + (inputs.mode == 'full' || inputs.mode == 'c13' || inputs.mode == 'recovery') environment: plananvil-codex runs-on: - self-hosted @@ -318,20 +319,28 @@ jobs: run: | set -euo pipefail cd "${QUALIFICATION_REPO}" - # Baseline 2.3 permits the narrowly gated C13 fallback in both full and - # C13-only modes. The harness still tries ephemeral first and ignores - # this permission unless the recognized parent-thread failure occurs. + # All selected modes retain ephemeral-first C13. Permission to retry + # never applies unless the recognized parent-thread failure occurs. qualification_args=(--allow-c13-non-ephemeral-fallback) if [ "${{ inputs.mode }}" = "c13" ]; then qualification_args+=(--only C13) fi set +e - python3 tools/live_codex_qualification_harness_v7.py \ - --root "${QUALIFICATION_REPO}" \ - --source-commit "${GITHUB_SHA}" \ - --run-id "${GITHUB_RUN_ID}" \ - --output "${QUALIFICATION_ARTIFACT}" \ - "${qualification_args[@]}" + if [ "${{ inputs.mode }}" = "recovery" ]; then + python3 tools/live_codex_qualification_recovery.py \ + --root "${QUALIFICATION_REPO}" \ + --source-commit "${GITHUB_SHA}" \ + --run-id "${GITHUB_RUN_ID}" \ + --output "${QUALIFICATION_ARTIFACT}" \ + "${qualification_args[@]}" + else + python3 tools/live_codex_qualification_harness_v7.py \ + --root "${QUALIFICATION_REPO}" \ + --source-commit "${GITHUB_SHA}" \ + --run-id "${GITHUB_RUN_ID}" \ + --output "${QUALIFICATION_ARTIFACT}" \ + "${qualification_args[@]}" + fi rc=$? set -e echo "exit_code=${rc}" >> "${GITHUB_OUTPUT}" diff --git a/.github/workflows/plananvil-tests.yml b/.github/workflows/plananvil-tests.yml index b69f26c..544e1ee 100644 --- a/.github/workflows/plananvil-tests.yml +++ b/.github/workflows/plananvil-tests.yml @@ -48,6 +48,8 @@ jobs: run: python -m compileall -q .agents/skills/plan-anvil .codex/hooks tools tests - name: Run core unit and integration tests run: python -m unittest discover -s .agents/skills/plan-anvil/tests -v + - name: Execute qualification boundary regressions without Codex + run: python -m unittest discover -s tests -p "test_qualification_*.py" -v - name: Check patch whitespace run: git diff --check diff --git a/docs/CODEX_QUALIFICATION_EXECUTION_AUDIT_2026-09-05.md b/docs/CODEX_QUALIFICATION_EXECUTION_AUDIT_2026-09-05.md new file mode 100644 index 0000000..fb468b1 --- /dev/null +++ b/docs/CODEX_QUALIFICATION_EXECUTION_AUDIT_2026-09-05.md @@ -0,0 +1,58 @@ +# Qualification execution-boundary audit — 2026-09-05 + +## Scope and authority + +Audited PlanAnvil source: `7b6322e4c2b11d4b78b5be714c94b88644fb6147`. +Observed live run: [qualification #21](https://github.com/KeyffMS/PlanAnvil/actions/runs/33968031850), Codex CLI `0.153.4`, explicit model `gpt-5.6-sol`. + +The implementation specification, generator/executor separation, product hooks, approval policy, sandbox, source immutability, baseline 2.3 and all `expected.json` assertions remain unchanged. This patch repairs qualification machinery, not the meaning of the product capabilities. Offline tests are not live evidence. + +## Source-verified contracts + +- [Official hooks documentation](https://developers.openai.com/codex/hooks): project/user hook sources, context output and event-specific control effects. +- [Pinned config loader](https://github.com/openai/codex/blob/rust-v0.153.4/codex-rs/config/src/loader/mod.rs): `root_checkout_hooks_folder_for_dir` and `merge_root_checkout_project_hooks` select root-checkout hook declarations for linked worktrees. Ordinary worktree-local config is distinct from hook-declaration provenance. +- [Pinned hook discovery](https://github.com/openai/codex/blob/rust-v0.153.4/codex-rs/hooks/src/engine/discovery.rs): uses each layer's `hooks_config_folder()`; hook sources can be additive. +- [Pinned hook execution](https://github.com/openai/codex/blob/rust-v0.153.4/codex-rs/hooks/src/engine/command_runner.rs): executes commands with the event cwd, which is not necessarily the directory containing the declarations. +- [Pinned startup semantics](https://github.com/openai/codex/blob/rust-v0.153.4/codex-rs/hooks/src/events/session_start.rs): SubagentStart matches `agent_type`, injects context and does not use `continue:false` as a startup stop. + +## Defect, repair and executable regression + +### C13 process contract + +The active compatibility context substituted a generic proxy requiring event and script arguments while the project fixture generated a command with neither argument. The proxy raised `IndexError` before executing the actual hook or recording telemetry. Zero recorder entries did not prove missing project-hook discovery. + +Repair: pass `SubagentStart subagent-start-fixture.py` to the existing generated command. The agent and hook remain project-scoped, with explicit `fixture_agent`, and the recognized-error-only ephemeral fallback is unchanged. + +Regression: enter the real compatibility context, seed the same declared project fixture as v7, execute the actual configured command in a repository with spaces, and assert exit code, context JSON, continue=false, one telemetry record and unchanged Git state. No Codex process or model is involved in this process-contract test. + +### C10 configuration provenance + +The previous PostCompact probe removed SessionStart only from a linked planning worktree. Codex still loaded the primary checkout's declarations. The same proof could reach the model through SessionStart, invalidating attribution to PostCompact. + +Repair: independent source repositories, planning worktrees and random proofs for the two trials. Prepare the second root checkout's hook selection before the fixture commit, product bootstrap and checkpoint. Verify root/local declarations agree and preserve both source and planning state during the probe. Keep the actual product recovery and compaction scripts; never synthesize a live hook event in the qualification runtime. + +Regression: the offline driver substitutes only Codex. It executes the real installer, Git operations, product start/checkpoint/validator and generated hook processes, using root-checkout declarations and planning cwd. It checks independent roots/proofs, valid checkpoints, context delivery and no proof retention. The driver models the pinned loader rule; it is not a substitute for live confirmation of that rule. + +### C09 successful completion + +Run #21 recorded 31 PreCompact and 30 PostCompact events and a Codex invocation timeout. The old evaluator could nevertheless classify the trial REPRODUCED. + +Repair: require successful invocation completion and a positive structured C09 result in addition to the existing two-cycle, checkpoint and continuation assertions. A timeout or missing result is BLOCKED, never a pass. Partial lifecycle observations remain visible. C08's intentional negative stop trial is not changed. + +Regression: call the actual C09 evaluator with controlled observations; two cycles cannot conceal a timeout or absent completion payload. Completed positive evidence still passes; a missing continuation or observed stop still prevents reproduction. + +This patch does not guess a new auto-compaction threshold. C09 may still time out live; that would now be accurately reported rather than converted into a green result. Repeated compaction under the artificially low fixture limit is not, by itself, proof of a product loop. + +## Red-to-green verification + +The test-only commit `f25fb15e4664a7afab371f696eae71dedd8876fe` added executable regressions before repairs. [Hosted CI #102](https://github.com/KeyffMS/PlanAnvil/actions/runs/33976076211) ran 134 distribution/harness tests and reported exactly four failures: C13 argv, C10 root-source isolation, C09 timeout and C09 missing output. Existing tests did not detect those defects. + +The same regressions remain in the patched suite. CI additionally executes qualification boundary tests on Linux, macOS and Windows with Python 3.11 and 3.14. The POSIX platforms execute the exact generated shell command; Windows checks the equivalent generated Python argv contract. The actual controlled live runner is Linux. + +## Targeted runner handoff + +Use the existing allowed workflow `PlanAnvil Codex qualification`, branch `main`, mode `recovery`. It selects C09, C10 and C13 using the same v7 capability runtimes as `full`; it does not use the separate precision fixtures. + +The targeted summary explicitly declares `scope`, `diagnostic_only=true`, `selected_gate_passed` and `release_gate_passed=false`. The full index still considers every required capability. A targeted success cannot release the product or substitute for a final C01–C16 run. + +No hosted test needs runner credentials, and this change does not dispatch a self-hosted run. No claim is made that the upstream ephemeral parent-thread failure is fixed. Live qualification remains necessary after offline verification. diff --git a/tests/test_live_codex_qualification_c10.py b/tests/test_live_codex_qualification_c10.py index 2fe4429..3dafa9c 100644 --- a/tests/test_live_codex_qualification_c10.py +++ b/tests/test_live_codex_qualification_c10.py @@ -1,8 +1,10 @@ from __future__ import annotations from contextlib import nullcontext +import json from pathlib import Path import sys +import tempfile import types import unittest from unittest import mock @@ -26,17 +28,28 @@ def test_c10_fixture_is_outer_harness_owned_and_product_validated(self) -> None: source = C10_SOURCE.read_text(encoding="utf-8") self.assertIn("v4._start_active_run(", source) self.assertIn("v4._create_checkpoint(planning=planning, run_root=run_root)", source) - self.assertGreaterEqual(source.count("v4._checkpoint_validation(planning)"), 4) + self.assertEqual(source.count("v4._checkpoint_validation(planning)"), 2) + self.assertEqual(source.count("v4._checkpoint_validation(compact_planning)"), 2) self.assertIn('"fixture_prepared_by_outer_harness=true"', source) self.assertNotIn("planner_prompt", source) def test_postcompact_is_independent_of_session_start_context(self) -> None: - source = C10_SOURCE.read_text(encoding="utf-8") - self.assertIn('configured.pop("SessionStart", None)', source) - self.assertIn('bool(configured.get("PreCompact"))', source) - self.assertIn('bool(configured.get("PostCompact"))', source) - self.assertIn("not compact_session_start", source) - self.assertIn("postcompact_isolated", source) + with tempfile.TemporaryDirectory() as tmp: + repo = Path(tmp) + hooks = {"hooks": {name: [{"hooks": [{"type": "command", "command": "fixture"}]}] + for name in ("SessionStart", "PreCompact", "PostCompact")}} + c10.base.json_dump(repo / ".codex/hooks.json", hooks) + self.assertTrue(c10._disable_session_start_for_postcompact(repo)) + remaining = c10.base.load_json(repo / ".codex/hooks.json")["hooks"] + self.assertNotIn("SessionStart", remaining) + self.assertEqual(remaining["PreCompact"], hooks["hooks"]["PreCompact"]) + self.assertEqual(remaining["PostCompact"], hooks["hooks"]["PostCompact"]) + + def test_missing_compaction_handler_fails_isolation(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + repo = Path(tmp) + c10.base.json_dump(repo / ".codex/hooks.json", {"hooks": {"SessionStart": []}}) + self.assertFalse(c10._disable_session_start_for_postcompact(repo)) def test_compaction_trigger_is_qualification_only(self) -> None: source = C10_SOURCE.read_text(encoding="utf-8") @@ -46,12 +59,14 @@ def test_compaction_trigger_is_qualification_only(self) -> None: self.assertIn("C10_COMPACT_LIMIT = 200", source) def test_opaque_recovery_value_is_not_persisted_in_evidence(self) -> None: - source = C10_SOURCE.read_text(encoding="utf-8") - self.assertIn("proof = secrets.token_hex(16)", source) - self.assertIn("def _payload_summary", source) - self.assertIn('"opaque_recovery_value_persisted=false"', source) - self.assertNotIn('"model_payload": session_payload', source) - self.assertNotIn('"model_payload": compact_payload', source) + secret_a, secret_b = "a" * 32, "b" * 32 + raw = {"error": "failure " + secret_a, "trials": [{"detail": "target=" + secret_b}]} + redacted = c10._redact_proofs(raw, (secret_a, secret_b)) + self.assertNotIn(secret_a, json.dumps(redacted)) + self.assertNotIn(secret_b, json.dumps(redacted)) + self.assertIn(secret_a, raw["error"]) + self.assertFalse(c10._exact_echo({"observations": ["C10_RECOVERY_ECHO=" + secret_b]}, secret_a)) + self.assertTrue(c10._exact_echo({"observations": ["C10_RECOVERY_ECHO=" + secret_a]}, secret_a)) def test_install_routes_only_c10(self) -> None: calls: list[dict[str, object]] = [] diff --git a/tests/test_prepare_capabilities_overlay.py b/tests/test_prepare_capabilities_overlay.py index 0bebd9d..25eae04 100644 --- a/tests/test_prepare_capabilities_overlay.py +++ b/tests/test_prepare_capabilities_overlay.py @@ -44,13 +44,14 @@ def test_c13_baseline23_overlay_materializes_and_rehashes(self) -> None: self.assertIn("agent_type", readme) self.assertIn("fixture_agent", readme) self.assertIn( - "live_codex_qualification_harness_v6.py", + "live_codex_qualification_harness_v7.py", (c13 / "run-command.txt").read_text(encoding="utf-8"), ) config = (c13 / "config" / "README.md").read_text(encoding="utf-8") - self.assertIn("home-scoped", config) + self.assertIn("No home-scoped synthetic agent or hook substitutes", config) self.assertIn("project-scoped", config) - self.assertIn("fixture_agent.toml", config) + self.assertIn("[agents.fixture_agent]", config) + self.assertIn('config_file = "./agents/fixture_agent.toml"', config) self.assertIn("agent_type=fixture_agent", config) prompt = (c13 / "prompt.txt").read_text(encoding="utf-8") self.assertIn("agent_type` exactly `fixture_agent", prompt) @@ -68,6 +69,27 @@ def test_c13_baseline23_overlay_materializes_and_rehashes(self) -> None: self.assertEqual(index["baseline_version"], "2.3") self.assertEqual(validate_capabilities.validate_all(target), []) + def test_recovery_overlays_do_not_change_expected_assertions(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) + prepare_capabilities.materialize(ROOT, target, force=True) + expected = { + "C09": ["Valid checkpoint allows compaction.", + "Recovery reconciles canonical files/Git after compaction.", + "A second valid compaction path is not permanently blocked."], + "C10": ["Recovery hook injects a pointer/context, not hidden mutable state.", + "Session continuation can reconstruct from canonical files and Git."], + } + for cid, assertions in expected.items(): + directory = target / "capabilities" / cid + package = json.loads((directory / "expected.json").read_text(encoding="utf-8")) + self.assertEqual(package["assertions"], assertions) + self.assertIn("live_codex_qualification_recovery.py", + (directory / "run-command.txt").read_text(encoding="utf-8")) + self.assertIn("BEFORE", (target / "capabilities/C10/fixture/README.md").read_text(encoding="utf-8")) + self.assertIn("timeout", (target / "capabilities/C09/fixture/README.md").read_text(encoding="utf-8")) + self.assertEqual(validate_capabilities.validate_all(target), []) + def test_overlays_do_not_remove_other_capabilities(self) -> None: with tempfile.TemporaryDirectory() as tmp: target = Path(tmp) / "materialized" diff --git a/tests/test_qualification_execution_boundaries.py b/tests/test_qualification_execution_boundaries.py index 57bfa7d..0a02178 100644 --- a/tests/test_qualification_execution_boundaries.py +++ b/tests/test_qualification_execution_boundaries.py @@ -215,7 +215,7 @@ def driver(args, *, cwd, **kwargs): self.assertIn("PostCompact", configured_events[1]) self.assertNotEqual(proofs[0], proofs[1], "Recovery probes must have independent proof values") for proof in proofs: - self.assertNotIn(proof, json.dumps(evidence), "Opaque proof leaked into persisted evidence") + self.assertNotIn(proof, json.dumps(evidence, default=str), "Opaque proof leaked into persisted evidence") if __name__ == "__main__": diff --git a/tests/test_qualification_recovery.py b/tests/test_qualification_recovery.py new file mode 100644 index 0000000..5cb7467 --- /dev/null +++ b/tests/test_qualification_recovery.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from contextlib import ExitStack +import io +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) +import live_codex_qualification_recovery as recovery + + +class TargetedRecoveryTests(unittest.TestCase): + def test_partial_scope_never_opens_release_gate(self) -> None: + for failed in (None, "C09", "C10", "C13"): + with self.subTest(failed=failed): + results = {cid: "BLOCKED" if cid == failed else "REPRODUCED" for cid in recovery.SCOPE} + summary = recovery.selected_summary(results) + self.assertIs(summary["release_gate_passed"], False) + self.assertEqual(summary["selected_gate_passed"], failed is None) + self.assertFalse(recovery.selected_summary({})["selected_gate_passed"]) + + def test_controller_uses_existing_runtime_and_continues_after_blocker(self) -> None: + with tempfile.TemporaryDirectory() as tmp, ExitStack() as stack: + root = Path(tmp) / "repo" + (root / ".git/info").mkdir(parents=True) + stack.enter_context(mock.patch.object(recovery.v7.prior, "_prepare_controller_root")) + stack.enter_context(mock.patch.object(recovery.base, "codex_version", return_value="offline-test")) + stack.enter_context(mock.patch.object(recovery.v7, "_install")) + stack.enter_context(mock.patch("sys.stdout", new_callable=io.StringIO)) + observed = [] + + def runtime(**kw): + observed.append(kw["capability_id"]) + self.assertTrue(recovery.v7.v6.ALLOW_NON_EPHEMERAL_FALLBACK) + if kw["capability_id"] == "C10": + raise RuntimeError("controlled offline fault") + return "REPRODUCED", True + + stack.enter_context(mock.patch.object(recovery.v7.v6, "capability_runtime", side_effect=runtime)) + evidence = stack.enter_context(mock.patch.object(recovery.base, "write_evidence")) + stack.enter_context(mock.patch.object(recovery.base, "local_commit")) + index = stack.enter_context(mock.patch.object(recovery.base, "finalize_index")) + artifact = stack.enter_context(mock.patch.object(recovery.base, "stage_artifact")) + previous = recovery.v7.v6.ALLOW_NON_EPHEMERAL_FALLBACK + rc = recovery.main([ + "--root", str(root), "--source-commit", "a" * 40, + "--run-id", "offline", "--output", str(Path(tmp) / "artifact"), + "--allow-c13-non-ephemeral-fallback", + ]) + self.assertEqual(rc, 2) + self.assertEqual(observed, ["C09", "C10", "C13"]) + self.assertEqual(evidence.call_args.kwargs["capability_id"], "C10") + self.assertEqual(evidence.call_args.kwargs["result"], "BLOCKED") + self.assertEqual(index.call_args.kwargs["results"]["C10"], "BLOCKED") + self.assertFalse(artifact.call_args.args[2]["release_gate_passed"]) + self.assertFalse((root / ".qualification-runtime").exists()) + self.assertEqual(recovery.v7.v6.ALLOW_NON_EPHEMERAL_FALLBACK, previous) + + def test_live_trust_restores_config_on_exception_without_restoring_auth(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + home = Path(tmp) / "home" + repo = Path(tmp) / "repo" + home.mkdir() + repo.mkdir() + original = b'model = "offline"\n' + config = home / "config.toml" + config.write_bytes(original) + auth = home / "auth.json" + auth.write_text("original-auth", encoding="utf-8") + common = recovery.base.common_codex_args + with mock.patch.dict(os.environ, {"CODEX_HOME": str(home)}): + with self.assertRaisesRegex(RuntimeError, "controlled"): + with recovery.v7._live_runner_persisted_trust_runtime(): + recovery.base.common_codex_args(cwd=repo, sandbox="read-only", + schema=repo / "schema.json", output=repo / "out.json") + auth.write_text("refreshed-auth", encoding="utf-8") + raise RuntimeError("controlled") + self.assertEqual(os.environ["CODEX_HOME"], str(home)) + self.assertEqual(config.read_bytes(), original) + self.assertEqual(auth.read_text(encoding="utf-8"), "refreshed-auth") + self.assertIs(recovery.base.common_codex_args, common) + + def test_recovery_workflow_keeps_existing_full_entrypoint(self) -> None: + text = (ROOT / ".github/workflows/plananvil-codex-qualification.yml").read_text(encoding="utf-8") + self.assertIn(" - recovery", text) + full = text[text.index(" full:"):] + self.assertIn("inputs.mode == 'recovery'", full) + self.assertIn("python3 tools/live_codex_qualification_recovery.py", full) + self.assertIn("python3 tools/live_codex_qualification_harness_v7.py", full) + self.assertIn("qualification_args=(--allow-c13-non-ephemeral-fallback)", full) + self.assertIn("refs/heads/main", full) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/live_codex_qualification_recovery.py b/tools/live_codex_qualification_recovery.py new file mode 100644 index 0000000..076a6e0 --- /dev/null +++ b/tools/live_codex_qualification_recovery.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import argparse +import datetime as dt +import json +import shutil +from pathlib import Path + +import live_codex_qualification_harness_v7 as v7 + +base = v7.base +SCOPE = ("C09", "C10", "C13") + + +def selected_summary(results: dict[str, str]) -> dict: + """A successful targeted run is never a successful full release gate.""" + missing = [cid for cid in SCOPE if results.get(cid) != "REPRODUCED"] + return { + "scope": list(SCOPE), + "diagnostic_only": True, + "results": {cid: results.get(cid, "BLOCKED") for cid in SCOPE}, + "selected_not_reproduced": missing, + "selected_gate_passed": not missing, + "release_gate_passed": False, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Run the existing C09/C10/C13 live runtimes only") + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--source-commit", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--allow-c13-non-ephemeral-fallback", action="store_true", required=True) + args = parser.parse_args(argv) + root = args.root.resolve() + output = args.output.resolve() + if output == root or output in root.parents or output.is_relative_to(root): + raise base.QualificationError("Evidence output must be separate from the qualification repository") + + v7.prior._prepare_controller_root(root, args.source_commit) + version = base.codex_version() + os_name = base.os_label() + date = dt.date.today().isoformat() + runtime_root = root / ".qualification-runtime" + shutil.rmtree(runtime_root, ignore_errors=True) + runtime_root.mkdir() + exclude = root / ".git" / "info" / "exclude" + with exclude.open("a", encoding="utf-8") as handle: + handle.write("\n.qualification-runtime/\n") + + old_fallback = v7.v6.ALLOW_NON_EPHEMERAL_FALLBACK + results: dict[str, str] = {} + try: + schemas = base.write_schemas(runtime_root / "schemas") + v7._install() + v7.v6.ALLOW_NON_EPHEMERAL_FALLBACK = args.allow_c13_non_ephemeral_fallback + for cid in SCOPE: + print(f"=== {cid}: targeted live qualification ===", flush=True) + try: + result, _required = v7.v6.capability_runtime( + root=root, runtime_root=runtime_root, capability_id=cid, + schemas=schemas, version=version, os_name=os_name, + source_commit=args.source_commit, date=date, + ) + if result not in {"REPRODUCED", "FAILED", "BLOCKED"}: + raise base.QualificationError("Capability runtime returned an invalid classification") + except Exception as exc: + # Same evidence contract as full. Continue to the other selected + # capabilities; an incomplete probe never becomes a release pass. + blocker = base.sanitize_text(f"{type(exc).__name__}: {exc}") + base.write_evidence( + root=root, capability_id=cid, result="BLOCKED", expected_met=False, + observations=["Targeted qualification controller did not complete this capability."], + blocker=blocker, summary=f"{cid} blocked by qualification controller error.", + trials=[], fixture_commit=None, version=version, os_name=os_name, + source_commit=args.source_commit, date=date, + ) + base.local_commit(root, cid) + result = "BLOCKED" + results[cid] = result + print(f"{cid}: {result}", flush=True) + + # finalize_index considers ALL required capabilities, not merely SCOPE. + base.finalize_index(root, date=date, source_commit=args.source_commit, + run_id=args.run_id, results=results) + summary = { + "schema_version": "1.0", "date": date, + "source_commit": args.source_commit, "github_actions_run": args.run_id, + "codex_version": version, "model": base.MODEL, "os": os_name, + **selected_summary(results), + } + base.stage_artifact(root, output, summary) + print(json.dumps(summary, indent=2, sort_keys=True), flush=True) + return 0 if summary["selected_gate_passed"] else 2 + finally: + v7.v6.ALLOW_NON_EPHEMERAL_FALLBACK = old_fallback + shutil.rmtree(runtime_root, ignore_errors=True) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/prepare_capabilities.py b/tools/prepare_capabilities.py index c136437..fabf184 100644 --- a/tools/prepare_capabilities.py +++ b/tools/prepare_capabilities.py @@ -48,91 +48,122 @@ - Release-gating: `yes` - Current result: `BLOCKED` - Qualification package state: `READY_FOR_LIVE_RUN` -- Prepared: `2026-09-03` +- Prepared: `2026-09-05` - Baseline: `2.3` -- Target runtime: Codex CLI `0.152.x` +- Target runtime: Codex CLI `0.153.4`; record the exact executed version ## Objective -Verify real `SubagentStart` context injection and the documented non-blocking meaning of `continue=false` without conflating those semantics with independent ephemeral parent-thread registration or project-scoped synthetic-agent discovery limitations. +Verify real project-scoped `SubagentStart` context injection and the documented non-blocking meaning of `continue=false`, separately from the known ephemeral parent-thread failure and from errors in the qualification proxy process. -Codex 0.152 matches `SubagentStart` handlers against the spawned `agent_type`. The qualification child must therefore be spawned with `agent_type` exactly `fixture_agent`; a default or unnamed child is not equivalent. +Codex matches `SubagentStart` handlers against the spawned `agent_type`. The qualification child must therefore be spawned with `agent_type` exactly `fixture_agent`; a default or unnamed child is not equivalent. ## Baseline 2.3 transport -The live harness attempts the aligned project-scoped `fixture_agent` through `codex exec --ephemeral` first. Only the recognized `collab spawn failed: no thread with id` failure may activate a controlled non-ephemeral retry. That retry uses a separate disposable repository containing the project-scoped hook/config but no project-scoped synthetic agent; the child is materialized as `CODEX_HOME/agents/fixture_agent.toml` inside a disposable `CODEX_HOME`. +The v7 live harness attempts the explicitly declared project-scoped `fixture_agent` through `codex exec --ephemeral` first. Only the recognized `collab spawn failed: no thread with id` failure may activate a controlled non-ephemeral retry. That retry uses a separate disposable repository containing both the project-scoped agent and the project-scoped hook. Its disposable `CODEX_HOME` supplies persisted project trust and isolated runtime persistence, not a substitute agent or hook. + +`REPRODUCED` requires one real project-scoped `SubagentStart`, `additionalContext`, a child echo of an outer-generated proof absent from the root prompt, unchanged repository state, verified session cleanup, and unchanged authentication metadata. `continue=false` is recorded as a compatibility signal but is not expected to stop `SubagentStart`. -`REPRODUCED` still requires one real project-scoped `SubagentStart`, `additionalContext`, a child echo of an outer-generated proof absent from the root prompt, unchanged repository state, verified session cleanup, and unchanged authentication metadata. `continue=false` is recorded as a compatibility signal but is not expected to stop `SubagentStart` on Codex 0.152. +The generated qualification proxy command must pass both required arguments: `SubagentStart subagent-start-fixture.py`. Offline process tests verify this contract but never constitute live capability evidence. ## Live metadata to record -Before changing this result to `REPRODUCED`, record the exact Codex version, model slug, OS, permission mode, project trust, fixture commit, transport used, exact requested `agent_type`, setup/cleanup, sanitized observations, evaluation, and hashes. Do not commit transcripts, credentials, private paths, proof values, session IDs, or unrelated repository data. +Record the exact Codex version, model slug, OS, permission mode, persisted project trust, fixture commit, transport used, exact requested `agent_type`, setup/cleanup, sanitized observations, evaluation, and hashes. Do not commit transcripts, credentials, private paths, proof values, session IDs, or unrelated repository data. ''', 'fixture/README.md': '''# C13 fixture -The deterministic harness owns this synthetic fixture. - -Ephemeral attempt: -- project-scoped agent file `.codex/agents/fixture_agent.toml`; -- declared agent name `fixture_agent`; +Both the ephemeral attempt and the recognized-error-only fallback use: +- project-scoped agent `.codex/agents/fixture_agent.toml`; +- explicit `[agents.fixture_agent]` with `config_file = "./agents/fixture_agent.toml"`; - spawn request with `agent_type` exactly `fixture_agent`; -- project-scoped `SubagentStart` hook matcher `^fixture_agent$`. - -Recognized-error fallback only: -- separate disposable Git repository with the same project-scoped hook/config; -- no project-scoped `.codex/agents` child definition; -- synthetic child materialized only as `CODEX_HOME/agents/fixture_agent.toml` inside a disposable `CODEX_HOME`; -- spawn request still uses `agent_type` exactly `fixture_agent`; -- sandbox remains read-only and repository state must remain unchanged. +- project-scoped `SubagentStart` matcher `^fixture_agent$`; +- a proxy command with arguments `SubagentStart subagent-start-fixture.py`. + +The fallback uses a separate disposable Git repository and an isolated CODEX_HOME. It does not materialize a home-scoped agent or hook. The sandbox remains read-only and repository state must remain unchanged. No manual invocation of the fixture hook can count as live evidence. ''', 'fixture/agent-role.txt': '''Synthetic agent role: fixture_agent. -The real spawn request must set agent_type exactly to fixture_agent because Codex 0.152 uses agent_type as the SubagentStart matcher input. The ephemeral attempt is project-scoped. The recognized-error fallback materializes the same role only in disposable CODEX_HOME/agents while keeping SubagentStart hooks project-scoped. +The real spawn request must set agent_type exactly to fixture_agent. Both transports declare the role in the project and keep SubagentStart project-scoped. The child echoes only context it actually received; the opaque value is absent from its own instructions and from the root prompt. ''', 'config/README.md': '''# C13 sandbox configuration — baseline 2.3 -Use the deterministic live qualification harness rather than an interactive manual session. - -Common requirements: +Use the deterministic v7 live qualification harness rather than an interactive manual session. ```toml [agents] enabled = true max_concurrent_threads_per_session = 2 + +[agents.fixture_agent] +description = "C13 qualification child for real SubagentStart context semantics." +config_file = "./agents/fixture_agent.toml" ``` - model: `gpt-5.6-sol`; - approval: `never`; - sandbox: `read-only`; - model-tool network: disabled; -- trusted disposable Git repository; -- real project-scoped `SubagentStart` hook; -- aligned agent filename/name/matcher: `fixture_agent.toml` / `fixture_agent` / `^fixture_agent$`; +- project trust persisted in the disposable user config, never passed as a projects CLI override; +- real project-scoped agent and `SubagentStart` hook; +- filename/name/matcher: `fixture_agent.toml` / `fixture_agent` / `^fixture_agent$`; - real spawn request uses `agent_type=fixture_agent`. -Transport is ephemeral-first. A non-ephemeral retry is allowed only for the recognized parent-thread registration failure. The retry uses a disposable `CODEX_HOME`, home-scoped synthetic agent, temporary file-backed auth symlink, isolated SQLite/log paths, `history.persistence="none"`, mandatory cleanup, and auth-metadata verification. +Transport is ephemeral-first. A non-ephemeral retry is allowed only for the recognized parent-thread registration failure. The retry uses a disposable CODEX_HOME, temporary file-backed auth symlink, isolated SQLite/log paths, history.persistence="none", mandatory cleanup, and auth-metadata verification. No home-scoped synthetic agent or hook substitutes for the project integration. ''', - 'prompt.txt': '''Capability qualification C13: real SubagentStart context semantics on Codex 0.152. + 'prompt.txt': '''Capability qualification C13: real project-scoped SubagentStart context semantics. Start exactly one configured child through the real Codex `spawn_agent` mechanism with `agent_type` exactly `fixture_agent`. Do not omit agent_type, do not use a default/unnamed child, do not simulate the child, and do not invoke hook scripts manually. The root session must not use command/file mutation tools. -The real project-scoped SubagentStart hook injects an opaque proof that is not present in this prompt and returns `continue=false`. On Codex 0.152, continue=false is a compatibility signal for this event, not a stop control. Wait for the real child and preserve only the minimal structural result needed to establish whether it received and echoed the injected proof. +The real project-scoped SubagentStart hook injects an opaque proof that is not present in this prompt and returns `continue=false`. This is a compatibility signal for the event, not a stop control. Wait for the real child and preserve only the minimal structural result needed to establish whether it received and echoed the injected proof. Do not expose credentials, proof values, usernames, home directories, session/thread IDs, private repository URLs, or full transcripts. ''', - 'run-command.txt': '''# Preferred controlled execution from main: -# Actions -> PlanAnvil Codex qualification -> mode=full -# -# Equivalent controller invocation inside the trusted disposable qualification workspace: -python3 tools/live_codex_qualification_harness_v6.py \\ + 'run-command.txt': '''# Targeted validation: existing workflow, branch main, mode=recovery (C09/C10/C13). +# Full release qualification: the same workflow, mode=full, after targeted validation. +python3 tools/live_codex_qualification_harness_v7.py \\ --root \\ --source-commit \\ --run-id \\ --output \\ --allow-c13-non-ephemeral-fallback -# -# The permission flag does not force non-ephemeral execution. C13 always runs -# ephemeral first and activates the fallback only for the recognized parent-thread failure. +# C13 always runs ephemeral first. Only the recognized parent-thread failure +# may activate the project-scoped non-ephemeral fallback. +''', +} + +C09_COMPLETION_OVERLAY = { + 'fixture/README.md': '''# C09 fixture and completion requirements + +The outer harness installs the actual product, creates the planning worktree and a valid checkpoint, and then exercises genuine automatic compaction. + +Two real compaction cycles, coherent checkpoint/Git state, and subsequent real tool use remain required. They are not sufficient when Codex times out or fails to return a completed positive structured C09 result. Partial event counts cannot turn an incomplete invocation into REPRODUCED. + +The deliberately low fixture threshold is not a product default. This correction does not silently retune it or weaken C08's intentional negative stop trial. Record a remaining timeout as BLOCKED. +''', + 'run-command.txt': '''# Existing controlled workflow: main -> recovery for C09/C10/C13. +# The recovery driver selects the same v7 capability runtime used by full. +python3 tools/live_codex_qualification_recovery.py --root --source-commit --run-id --output --allow-c13-non-ephemeral-fallback +# A targeted pass is not a full C01-C16 release pass. +''', +} + +C10_ISOLATION_OVERLAY = { + 'fixture/README.md': '''# C10 independent recovery fixtures + +Prepare each fixture deterministically through the actual installer, product start command, checkpoint creator and checkpoint validator. The model must not construct its own prerequisites. + +SessionStart and PostCompact use independent source repositories, planning worktrees and opaque next-action targets. For PostCompact, remove SessionStart from the root checkout's hook declarations BEFORE the fixture commit and bootstrap. Codex 0.153.4 redirects linked-worktree hook declarations to the root checkout; changing only planning/.codex/hooks.json is not isolation. + +The live runtime must observe the actual product recovery hook and an exact opaque echo without unauthorized file/tool reads. Keep both source and planning state unchanged and redact proof values from persisted evidence. Offline command/lifecycle-driver tests verify setup, not live capability reproduction. +''', + 'config/README.md': '''# C10 configuration provenance + +Use the same v7 runner live-auth/persisted-trust context as C08/C09. Do not copy or restore authentication tokens. The runner config.toml is restored byte-for-byte after the probe. + +The SessionStart fixture retains the product startup hook. The independent PostCompact fixture retains PreCompact and PostCompact, excludes SessionStart at the primary hook source, and checks that the linked checkout has identical declarations. Source configuration is prepared before product snapshots/checkpoints, not mutated afterwards. + +Sandbox remains read-only, approval remains never, model-tool network access remains disabled. A low auto-compaction threshold and token_budget=false apply only to the disposable compaction fixture, not product defaults. ''', + 'run-command.txt': C09_COMPLETION_OVERLAY['run-command.txt'], } @@ -194,11 +225,11 @@ def materialize(source_root: Path, target_root: Path, *, force: bool = False) -> target.write_bytes(data) written.append(rel.as_posix()) - # The stable archive remains the historical prepared-package source. Small - # deterministic overlays keep runtime-sensitive release-gating capability - # documentation synchronized with the current product/harness contract and - # recompute package hashes before validation. + # Keep documentation synchronized without changing expected assertions or + # synthesizing live results. Recompute hashes before package validation. written.extend(_apply_overlay(target_root, 'C06', C06_CODEX0152_OVERLAY)) + written.extend(_apply_overlay(target_root, 'C09', C09_COMPLETION_OVERLAY)) + written.extend(_apply_overlay(target_root, 'C10', C10_ISOLATION_OVERLAY)) written.extend(_apply_overlay(target_root, 'C13', C13_BASELINE23_OVERLAY)) # The index and package guide are tracked outside the archive and are needed