From e655fbb38f2ee470c8c8289c6ca3a85c7f156f6b Mon Sep 17 00:00:00 2001 From: KeyffMS <124252104+KeyffMS@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:21:23 +0200 Subject: [PATCH 1/9] Use live auth and project-aligned C13 fallback --- tools/live_codex_qualification_harness_v7.py | 399 +++++++++++++++++++ 1 file changed, 399 insertions(+) create mode 100644 tools/live_codex_qualification_harness_v7.py diff --git a/tools/live_codex_qualification_harness_v7.py b/tools/live_codex_qualification_harness_v7.py new file mode 100644 index 0000000..f66441d --- /dev/null +++ b/tools/live_codex_qualification_harness_v7.py @@ -0,0 +1,399 @@ +from __future__ import annotations + +import contextlib +import os +import shutil +import stat +from pathlib import Path +from typing import Any, Iterator + +import live_codex_qualification_harness_v6 as v6 + +base = v6.base +compat = v6.compat +prior = v6.prior + + +def _runner_codex_home() -> Path: + raw = os.environ.get("CODEX_HOME") + return Path(raw).expanduser().resolve() if raw else (Path.home() / ".codex").resolve() + + +def _restore_file(path: Path, existed: bool, content: bytes, mode: int | None) -> None: + if existed: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + if mode is not None: + path.chmod(mode) + else: + path.unlink(missing_ok=True) + + +@contextlib.contextmanager +def _live_runner_persisted_trust_runtime() -> Iterator[Path]: + """Use the runner's real Codex auth while temporarily persisting project trust. + + C08/C09 run late enough in a full qualification that a copied/symlinked auth file can + become stale while the runner's real Codex home refreshes credentials. Keep the live + CODEX_HOME and change only config.toml, restoring it byte-for-byte afterwards. + """ + + home = _runner_codex_home() + config_path = home / "config.toml" + existed = config_path.exists() + content = config_path.read_bytes() if existed else b"" + mode = stat.S_IMODE(config_path.stat().st_mode) if existed else None + old_common = base.common_codex_args + previous_home = os.environ.get("CODEX_HOME") + + def common_codex_args(**kwargs: Any) -> list[str]: + cwd = Path(kwargs["cwd"]).resolve() + compat._write_persisted_project_trust(home, cwd) + kwargs["trust_project"] = False + args = old_common(**kwargs) + return [item for item in args if item != "--ignore-user-config"] + + base.common_codex_args = common_codex_args + try: + yield home + finally: + base.common_codex_args = old_common + if previous_home is None: + os.environ.pop("CODEX_HOME", None) + else: + os.environ["CODEX_HOME"] = previous_home + _restore_file(config_path, existed, content, mode) + restored = config_path.exists() == existed + if restored and existed: + restored = config_path.read_bytes() == content + if not restored: + raise base.QualificationError( + "Codex runner config.toml was not restored after persisted-trust qualification" + ) + + +def run_c08(**kwargs: Any): + cap_runtime = Path(kwargs["runtime_root"]) / "C08" + with ( + compat._codex0152_compaction(cap_runtime, "C08"), + _live_runner_persisted_trust_runtime(), + ): + return compat.v4._c08_runtime(**kwargs) + + +def run_c09(**kwargs: Any): + cap_runtime = Path(kwargs["runtime_root"]) / "C09" + with ( + compat._codex0152_compaction(cap_runtime, "C09"), + _live_runner_persisted_trust_runtime(), + ): + return compat.v4._c09_runtime(**kwargs) + + +def _seed_declared_project_fixture(repo: Path, context_proof: str) -> bool: + """Use the same project-scoped role/hook shape that PlanAnvil targets.""" + + v6._seed_project_fixture(repo, context_proof, include_project_agent=True) + config_path = repo / ".codex" / "config.toml" + text = config_path.read_text(encoding="utf-8") + header = f"[agents.{v6.HOME_AGENT_NAME}]" + if header not in text: + text = text.rstrip() + ( + f"\n\n{header}\n" + 'description = "C13 qualification child for real SubagentStart context semantics."\n' + f'config_file = "./agents/{v6.HOME_AGENT_FILENAME}"\n' + ) + v6._write(config_path, text.rstrip() + "\n") + return (repo / ".codex" / "agents" / v6.HOME_AGENT_FILENAME).is_file() + + +def _prepare_project_fallback_home( + cap_runtime: Path, + repo: Path, +) -> tuple[Path, Path | None, tuple[int, int, int] | None]: + """Persist trust in an isolated home but keep the fallback role and hook project-scoped.""" + + home, auth_path, auth_before = prior._prepare_isolated_codex_home( + cap_runtime / "project-non-ephemeral" + ) + compat._write_persisted_project_trust(home, repo) + return home, auth_path, auth_before + + +def _c13_runtime( + *, + root: Path, + runtime_root: Path, + schemas: dict[str, Path], + version: str, + os_name: str, + source_commit: str, + date: str, +) -> tuple[str, bool]: + capability_id = "C13" + ( + _cap_dir, + cap_runtime, + _spec_dir, + project_repo, + _worktrees, + results_dir, + _evaluator_dir, + ) = prior._runtime_paths(root=root, runtime_root=runtime_root, capability_id=capability_id) + fallback_repo = cap_runtime / "fallback-repo" + + with prior.prior.v2._python_bytecode_disabled(): + proof = prior._context_proof(source_commit) + + base.ensure_git_repo(project_repo) + _seed_declared_project_fixture(project_repo, proof) + project_fixture_commit = base.commit_fixture_baseline(project_repo) + + ephemeral_home, ephemeral_auth_path, ephemeral_auth_before = ( + v6._prepare_trusted_ephemeral_home(cap_runtime, project_repo) + ) + ephemeral_cleanup_verified = True + ephemeral_auth_unchanged = True + prior._clear_hook_log(project_repo) + before_ephemeral = base.git_snapshot(project_repo) + try: + payload_e, events_e, error_e, known_e = prior._run_c13_codex( + repo=project_repo, + schemas=schemas, + results_dir=results_dir, + position=1, + ephemeral=True, + isolated_codex_home=ephemeral_home, + timeout=600, + ) + after_ephemeral = base.git_snapshot(project_repo) + records_e = prior._read_hook_records(project_repo) + finally: + ephemeral_cleanup_verified, ephemeral_auth_unchanged = ( + prior._cleanup_isolated_codex_home( + ephemeral_home, + ephemeral_auth_path, + ephemeral_auth_before, + ) + ) + + outcome_e, trial_e = prior._evaluate_transport( + transport="ephemeral", + payload=payload_e, + events=events_e, + error=error_e, + known_parent_failure=known_e, + records=records_e, + context_proof=proof, + git_before=before_ephemeral, + git_after=after_ephemeral, + ) + if known_e: + outcome_e = "BLOCKED" + trial_e["outcome"] = "BLOCKED" + trial_e["blocker"] = "recognized ephemeral parent-thread registration failure" + elif not (ephemeral_cleanup_verified and ephemeral_auth_unchanged): + outcome_e = "BLOCKED" + trial_e["outcome"] = "BLOCKED" + trial_e["blocker"] = "isolated ephemeral CODEX_HOME cleanup/auth invariants failed" + trial_e["agent_fixture_scope"] = "project" + trial_e["agent_role_declared_explicitly"] = True + trial_e["required_spawn_agent_type"] = v6.HOME_AGENT_NAME + trial_e["persisted_project_trust"] = True + trial_e["isolated_codex_home"] = True + trial_e["isolated_home_cleanup_verified"] = ephemeral_cleanup_verified + trial_e["auth_metadata_unchanged"] = ephemeral_auth_unchanged + + trials: list[dict[str, Any]] = [base.sanitize(trial_e)] + fallback_used = False + fallback_available = False + cleanup_verified = True + auth_unchanged = True + project_agent_materialized = False + fallback_fixture_commit: str | None = None + + if outcome_e == "PASS": + final_outcome = "PASS" + transport_resolution = "ephemeral" + elif outcome_e == "FAILED": + final_outcome = "FAILED" + transport_resolution = "ephemeral_semantic_failure" + elif not (known_e and v6.ALLOW_NON_EPHEMERAL_FALLBACK): + final_outcome = "BLOCKED" + transport_resolution = ( + "ephemeral_known_transport_blocker_fallback_not_enabled" + if known_e + else "ephemeral_unclassified_blocker" + ) + else: + fallback_used = True + base.ensure_git_repo(fallback_repo) + project_agent_materialized = _seed_declared_project_fixture(fallback_repo, proof) + fallback_fixture_commit = base.commit_fixture_baseline(fallback_repo) + + isolated_home, auth_path, auth_before = _prepare_project_fallback_home( + cap_runtime, fallback_repo + ) + fallback_available = auth_path is not None and project_agent_materialized + + if not fallback_available: + shutil.rmtree(isolated_home, ignore_errors=True) + cleanup_verified = not isolated_home.exists() + final_outcome = "BLOCKED" + transport_resolution = "project_non_ephemeral_fallback_preflight_unavailable" + trials.append( + { + "capability_id": capability_id, + "trial": "non_ephemeral_project_agent_fallback_preflight", + "trial_name": "non_ephemeral_project_agent_fallback_preflight", + "transport": "non-ephemeral", + "outcome": "BLOCKED", + "assertions": [], + "observations": [ + f"project_agent_materialized={str(project_agent_materialized).lower()}", + f"file_backed_auth_bridge_available={str(auth_path is not None).lower()}", + "persisted_project_trust=true", + f"session_cleanup_verified={str(cleanup_verified).lower()}", + ], + "blocker": "project-scoped non-ephemeral fallback preflight unavailable", + } + ) + else: + prior._clear_hook_log(fallback_repo) + before_fallback = base.git_snapshot(fallback_repo) + payload_n: dict[str, Any] = {} + events_n: dict[str, Any] = {} + error_n: str | None = None + known_n = False + session_rollouts = 0 + after_fallback = before_fallback + records_n: list[dict[str, Any]] = [] + try: + payload_n, events_n, error_n, known_n = prior._run_c13_codex( + repo=fallback_repo, + schemas=schemas, + results_dir=results_dir, + position=2, + ephemeral=False, + isolated_codex_home=isolated_home, + timeout=600, + ) + session_rollouts = prior._session_rollout_count(isolated_home) + after_fallback = base.git_snapshot(fallback_repo) + records_n = prior._read_hook_records(fallback_repo) + finally: + cleanup_verified, auth_unchanged = prior._cleanup_isolated_codex_home( + isolated_home, + auth_path, + auth_before, + ) + + outcome_n, trial_n = prior._evaluate_transport( + transport="non-ephemeral", + payload=payload_n, + events=events_n, + error=error_n, + known_parent_failure=known_n, + records=records_n, + context_proof=proof, + git_before=before_fallback, + git_after=after_fallback, + session_rollouts_created=session_rollouts, + session_cleanup_verified=cleanup_verified, + auth_unchanged=auth_unchanged, + ) + trial_n["agent_fixture_scope"] = "project" + trial_n["project_agent_present"] = True + trial_n["project_agent_declared_explicitly"] = True + trial_n["home_agent_materialized"] = False + trial_n["project_scoped_subagent_start_hook"] = True + trial_n["required_spawn_agent_type"] = v6.HOME_AGENT_NAME + trial_n["fallback_fixture_commit"] = fallback_fixture_commit + trial_n["persisted_project_trust"] = True + trials.append(base.sanitize(trial_n)) + final_outcome = outcome_n + transport_resolution = "non_ephemeral_project_agent_fallback" + + if final_outcome == "PASS": + result = "REPRODUCED" + expected_met = True + blocker = None + if transport_resolution == "ephemeral": + summary = ( + "C13 reproduced directly under ephemeral execution with one real " + "SubagentStart hook and child context echo." + ) + else: + summary = ( + "C13 reproduced through the controlled non-ephemeral project-agent " + "fallback after the recognized ephemeral parent-thread registration " + "failure; role and SubagentStart hook remained project-scoped." + ) + elif final_outcome == "FAILED": + result = "FAILED" + expected_met = False + blocker = ( + "C13 reached real SubagentStart startup but observed semantics contradicted " + "the expected context/continue=false contract." + ) + summary = "C13 failed after reaching the real SubagentStart semantic boundary." + else: + result = "BLOCKED" + expected_met = False + blocker = ( + "C13 could not reach and verify the real project-scoped child startup semantics " + "under the permitted Codex 0.152 qualification transport." + ) + summary = ( + "C13 remains blocked because the real project-scoped SubagentStart semantic " + "boundary was not completely exercised." + ) + + evidence_fixture_commit = ( + fallback_fixture_commit if fallback_used and fallback_fixture_commit else project_fixture_commit + ) + return prior.prior._write_result( + root=root, + cap_runtime=cap_runtime, + capability_id=capability_id, + result=result, + expected_met=expected_met, + observations=[ + f"ephemeral_outcome={outcome_e}", + f"ephemeral_known_parent_thread_failure={str(known_e).lower()}", + "ephemeral_persisted_project_trust=true", + f"ephemeral_cleanup_verified={str(ephemeral_cleanup_verified).lower()}", + f"ephemeral_auth_metadata_unchanged={str(ephemeral_auth_unchanged).lower()}", + f"non_ephemeral_fallback_enabled={str(v6.ALLOW_NON_EPHEMERAL_FALLBACK).lower()}", + f"non_ephemeral_fallback_used={str(fallback_used).lower()}", + f"non_ephemeral_fallback_available={str(fallback_available).lower()}", + f"project_agent_materialized={str(project_agent_materialized).lower()}", + f"session_cleanup_verified={str(cleanup_verified).lower()}", + f"auth_metadata_unchanged={str(auth_unchanged).lower()}", + f"transport_resolution={transport_resolution}", + f"required_spawn_agent_type={v6.HOME_AGENT_NAME}", + ], + blocker=blocker, + summary=summary, + trials=trials, + fixture_commit=evidence_fixture_commit, + version=version, + os_name=os_name, + source_commit=source_commit, + date=date, + ) + + +def _install() -> None: + v6.compat.run_c08 = run_c08 + v6.compat.run_c09 = run_c09 + v6._c13_runtime = _c13_runtime + + +def main(argv: list[str] | None = None) -> int: + _install() + return v6.main(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) From 7120643483715d896eeec48032718685b19deb02 Mon Sep 17 00:00:00 2001 From: KeyffMS <124252104+KeyffMS@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:21:58 +0200 Subject: [PATCH 2/9] Run final Codex live harness --- .github/workflows/plananvil-codex-qualification.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/plananvil-codex-qualification.yml b/.github/workflows/plananvil-codex-qualification.yml index f89d3ed..d9f4d0b 100644 --- a/.github/workflows/plananvil-codex-qualification.yml +++ b/.github/workflows/plananvil-codex-qualification.yml @@ -326,7 +326,7 @@ jobs: qualification_args+=(--only C13) fi set +e - python3 tools/live_codex_qualification_harness_v6.py \ + python3 tools/live_codex_qualification_harness_v7.py \ --root "${QUALIFICATION_REPO}" \ --source-commit "${GITHUB_SHA}" \ --run-id "${GITHUB_RUN_ID}" \ From cedc670faaa1db959d21234c7a4d57d8c7c373bb Mon Sep 17 00:00:00 2001 From: KeyffMS <124252104+KeyffMS@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:22:21 +0200 Subject: [PATCH 3/9] Test final Codex live gate bridges --- ...est_live_codex_qualification_harness_v7.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 tests/test_live_codex_qualification_harness_v7.py diff --git a/tests/test_live_codex_qualification_harness_v7.py b/tests/test_live_codex_qualification_harness_v7.py new file mode 100644 index 0000000..d2a0050 --- /dev/null +++ b/tests/test_live_codex_qualification_harness_v7.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) + +import live_codex_qualification_harness_v7 as v7 + +WORKFLOW = ROOT / ".github" / "workflows" / "plananvil-codex-qualification.yml" +SOURCE = ROOT / "tools" / "live_codex_qualification_harness_v7.py" + + +class LiveCodexHarnessV7Tests(unittest.TestCase): + def test_full_workflow_uses_v7(self) -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + full = workflow[workflow.index(" full:"):] + self.assertIn("python3 tools/live_codex_qualification_harness_v7.py", full) + self.assertNotIn("python3 tools/live_codex_qualification_harness_v6.py", full) + + def test_live_runner_trust_restores_config_and_keeps_real_home(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + home = root / "home" + repo = root / "repo" + home.mkdir() + repo.mkdir() + config = home / "config.toml" + original = b'model = "fixture"\n' + config.write_bytes(original) + observed: list[dict[str, object]] = [] + + def fake_common(**kwargs: object) -> list[str]: + observed.append(dict(kwargs)) + return ["codex", "exec", "--ignore-user-config"] + + previous = os.environ.get("CODEX_HOME") + os.environ["CODEX_HOME"] = str(home) + try: + with ( + mock.patch.object(v7, "_runner_codex_home", return_value=home), + mock.patch.object(v7.base, "common_codex_args", side_effect=fake_common), + ): + with v7._live_runner_persisted_trust_runtime(): + args = v7.base.common_codex_args(cwd=repo, trust_project=True) + text = config.read_text(encoding="utf-8") + self.assertIn("[projects.", text) + self.assertIn('trust_level = "trusted"', text) + self.assertEqual(os.environ.get("CODEX_HOME"), str(home)) + self.assertNotIn("--ignore-user-config", args) + self.assertEqual(config.read_bytes(), original) + finally: + if previous is None: + os.environ.pop("CODEX_HOME", None) + else: + os.environ["CODEX_HOME"] = previous + + self.assertEqual(len(observed), 1) + self.assertIs(observed[0]["trust_project"], False) + + def test_c08_c09_use_live_auth_runtime(self) -> None: + source = SOURCE.read_text(encoding="utf-8") + self.assertIn("def run_c08", source) + self.assertIn("def run_c09", source) + self.assertGreaterEqual(source.count("_live_runner_persisted_trust_runtime()"), 2) + self.assertIn("change only config.toml", source) + + def test_c13_fallback_keeps_role_and_hook_project_scoped(self) -> None: + source = SOURCE.read_text(encoding="utf-8") + self.assertIn("_seed_declared_project_fixture(fallback_repo, proof)", source) + self.assertIn('trial_n["project_agent_present"] = True', source) + self.assertIn('trial_n["project_scoped_subagent_start_hook"] = True', source) + self.assertIn('trial_n["home_agent_materialized"] = False', source) + self.assertNotIn("_prepare_home_scoped_fallback_agent", source) + + def test_project_fixture_declares_exact_agent_role(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + repo = Path(tmp) + + def seed(target: Path, _proof: str, *, include_project_agent: bool) -> None: + self.assertTrue(include_project_agent) + (target / ".codex" / "agents").mkdir(parents=True) + (target / ".codex" / "config.toml").write_text( + "[agents]\nenabled = true\n", encoding="utf-8" + ) + (target / ".codex" / "agents" / v7.v6.HOME_AGENT_FILENAME).write_text( + 'name = "fixture_agent"\n', encoding="utf-8" + ) + + with mock.patch.object(v7.v6, "_seed_project_fixture", side_effect=seed): + materialized = v7._seed_declared_project_fixture(repo, "opaque") + config = (repo / ".codex" / "config.toml").read_text(encoding="utf-8") + self.assertTrue(materialized) + self.assertIn("[agents.fixture_agent]", config) + self.assertIn('config_file = "./agents/fixture_agent.toml"', config) + + def test_safety_boundaries_are_not_weakened(self) -> None: + source = SOURCE.read_text(encoding="utf-8") + self.assertNotIn("--dangerously-bypass-approvals-and-sandbox", source) + self.assertNotIn("danger-full-access", source) + self.assertNotIn("--privileged", source) + self.assertNotIn("SYS_ADMIN", source) + + +if __name__ == "__main__": + unittest.main() From dfaa1a9f0969f5b7d92ad09ef276c0227c8e52d7 Mon Sep 17 00:00:00 2001 From: KeyffMS <124252104+KeyffMS@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:24:25 +0200 Subject: [PATCH 4/9] Track v4 under current v7 wrapper --- tests/test_live_codex_qualification_harness_v4.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_live_codex_qualification_harness_v4.py b/tests/test_live_codex_qualification_harness_v4.py index f8daec5..235248d 100644 --- a/tests/test_live_codex_qualification_harness_v4.py +++ b/tests/test_live_codex_qualification_harness_v4.py @@ -55,16 +55,20 @@ def test_safety_boundary_is_not_weakened(self) -> None: self.assertIn('sandbox="read-only"', self.source) self.assertIn('sandbox="workspace-write"', self.source) - def test_v4_is_chained_under_current_v6_wrapper(self) -> None: + def test_v4_is_chained_under_current_v7_wrapper(self) -> None: v5 = (ROOT / "tools" / "live_codex_qualification_harness_v5.py").read_text( encoding="utf-8" ) v6 = (ROOT / "tools" / "live_codex_qualification_harness_v6.py").read_text( encoding="utf-8" ) + v7 = (ROOT / "tools" / "live_codex_qualification_harness_v7.py").read_text( + encoding="utf-8" + ) self.assertIn("import live_codex_qualification_harness_v4 as prior", v5) self.assertIn("import live_codex_qualification_harness_v5 as prior", v6) - self.assertIn("python3 tools/live_codex_qualification_harness_v6.py", self.workflow) + self.assertIn("import live_codex_qualification_harness_v6 as v6", v7) + self.assertIn("python3 tools/live_codex_qualification_harness_v7.py", self.workflow) if __name__ == "__main__": From 162f6b25c78e678f35515ad40a29eef6582e1eac Mon Sep 17 00:00:00 2001 From: KeyffMS <124252104+KeyffMS@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:24:45 +0200 Subject: [PATCH 5/9] Track v5 under current v7 transport --- tests/test_live_codex_qualification_harness_v5.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_live_codex_qualification_harness_v5.py b/tests/test_live_codex_qualification_harness_v5.py index bbf54ab..e48c18c 100644 --- a/tests/test_live_codex_qualification_harness_v5.py +++ b/tests/test_live_codex_qualification_harness_v5.py @@ -72,10 +72,10 @@ def test_safety_boundary_is_not_weakened(self) -> None: self.assertIn("completed_file_change_items", self.source) self.assertIn("repository_unchanged", self.source) - def test_current_workflow_uses_v6_and_keeps_c13_short_mode(self) -> None: + def test_current_workflow_uses_v7_and_keeps_c13_short_mode(self) -> None: self.assertIn("- c13", self.workflow) self.assertIn("inputs.mode == 'c13'", self.workflow) - self.assertIn("python3 tools/live_codex_qualification_harness_v6.py", self.workflow) + self.assertIn("python3 tools/live_codex_qualification_harness_v7.py", self.workflow) self.assertIn("--only C13", self.workflow) self.assertIn("--allow-c13-non-ephemeral-fallback", self.workflow) self.assertIn("inputs.mode == 'full'", self.workflow) @@ -83,9 +83,10 @@ def test_current_workflow_uses_v6_and_keeps_c13_short_mode(self) -> None: def test_runbook_documents_baseline23_full_transport(self) -> None: self.assertIn("mode=c13", self.runbook) self.assertIn("baseline 2.3", self.runbook.lower()) - self.assertIn("CODEX_HOME/agents/fixture_agent.toml", self.runbook) + self.assertIn("project-scoped synthetic agent", self.runbook) self.assertIn("project-scoped", self.runbook) self.assertIn("mode=full", self.runbook) + self.assertIn("live_codex_qualification_harness_v7.py", self.runbook) if __name__ == "__main__": From e7be83d765e88751118698a69d21116c6f022281 Mon Sep 17 00:00:00 2001 From: KeyffMS <124252104+KeyffMS@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:25:07 +0200 Subject: [PATCH 6/9] Track v6 beneath current v7 wrapper --- ...est_live_codex_qualification_harness_v6.py | 55 +++++++++---------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/tests/test_live_codex_qualification_harness_v6.py b/tests/test_live_codex_qualification_harness_v6.py index 4b3e6dc..17f88a6 100644 --- a/tests/test_live_codex_qualification_harness_v6.py +++ b/tests/test_live_codex_qualification_harness_v6.py @@ -6,6 +6,7 @@ ROOT = Path(__file__).resolve().parents[1] MODULE_PATH = ROOT / "tools" / "live_codex_qualification_harness_v6.py" +V7_PATH = ROOT / "tools" / "live_codex_qualification_harness_v7.py" COMPAT_PATH = ROOT / "tools" / "live_codex_qualification_codex0152.py" REGRESSION_PATH = ROOT / "tools" / "live_codex_qualification_regression.py" WORKFLOW_PATH = ROOT / ".github" / "workflows" / "plananvil-codex-qualification.yml" @@ -17,6 +18,7 @@ class LiveCodexHarnessV6Tests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.source = MODULE_PATH.read_text(encoding="utf-8") + cls.v7 = V7_PATH.read_text(encoding="utf-8") cls.compat = COMPAT_PATH.read_text(encoding="utf-8") cls.regression = REGRESSION_PATH.read_text(encoding="utf-8") cls.workflow = WORKFLOW_PATH.read_text(encoding="utf-8") @@ -51,34 +53,29 @@ def test_ephemeral_attempt_remains_project_scoped_and_persistently_trusted(self) self.assertIn("ephemeral_cleanup_verified", self.source) self.assertIn("ephemeral_auth_metadata_unchanged", self.source) - def test_fallback_separates_agent_discovery_from_project_hook(self) -> None: - self.assertIn("_seed_project_fixture(fallback_repo, proof, include_project_agent=False)", self.source) - self.assertIn('home / "agents" / HOME_AGENT_FILENAME', self.source) - self.assertIn("compat._write_persisted_project_trust(home, repo)", self.source) - self.assertIn("_declare_home_agent(home)", self.source) - self.assertIn("_prepare_home_scoped_fallback_agent(cap_runtime, fallback_repo)", self.source) - self.assertIn('trial_n["agent_fixture_scope"] = "disposable_CODEX_HOME"', self.source) - self.assertIn('trial_n["project_agent_present"] = False', self.source) - self.assertIn('trial_n["project_scoped_subagent_start_hook"] = True', self.source) - self.assertIn('trial_n["persisted_project_trust"] = True', self.source) - self.assertIn("compat.run_c13(_c13_runtime", self.source) - - def test_known_ephemeral_parent_failure_always_uses_the_gated_fallback(self) -> None: - self.assertIn('if known_e:\n outcome_e = "BLOCKED"', self.source) - self.assertIn("recognized ephemeral parent-thread registration failure", self.source) - self.assertIn("known_e and ALLOW_NON_EPHEMERAL_FALLBACK", self.source) - self.assertIn("ephemeral_known_transport_blocker_fallback_not_enabled", self.source) - self.assertIn("non_ephemeral_home_agent_fallback", self.source) + def test_v6_historical_fallback_is_superseded_by_v7_project_fallback(self) -> None: + self.assertIn("_prepare_home_scoped_fallback_agent", self.source) + self.assertIn("_seed_declared_project_fixture(fallback_repo, proof)", self.v7) + self.assertIn('trial_n["project_agent_present"] = True', self.v7) + self.assertIn('trial_n["project_scoped_subagent_start_hook"] = True', self.v7) + self.assertIn('trial_n["home_agent_materialized"] = False', self.v7) + + def test_known_ephemeral_parent_failure_remains_gated(self) -> None: + self.assertIn('if known_e:\n outcome_e = "BLOCKED"', self.v7) + self.assertIn("recognized ephemeral parent-thread registration failure", self.v7) + self.assertIn("v6.ALLOW_NON_EPHEMERAL_FALLBACK", self.v7) + self.assertIn("ephemeral_known_transport_blocker_fallback_not_enabled", self.v7) + self.assertIn("non_ephemeral_project_agent_fallback", self.v7) def test_non_ephemeral_cleanup_and_auth_invariants_remain_required(self) -> None: - self.assertIn("prior._prepare_isolated_codex_home", self.source) - self.assertIn("prior._cleanup_isolated_codex_home", self.source) - self.assertIn("session_cleanup_verified", self.source) - self.assertIn("auth_metadata_unchanged", self.source) - self.assertIn("home_scoped_fixture_agent_materialized", self.source) - - def test_full_workflow_stays_on_v6_and_enables_baseline23_fallback(self) -> None: - self.assertIn("python3 tools/live_codex_qualification_harness_v6.py", self.workflow) + self.assertIn("prior._prepare_isolated_codex_home", self.v7) + self.assertIn("prior._cleanup_isolated_codex_home", self.v7) + self.assertIn("session_cleanup_verified", self.v7) + self.assertIn("auth_metadata_unchanged", self.v7) + self.assertIn("project_agent_materialized", self.v7) + + def test_full_workflow_uses_v7_and_enables_baseline23_fallback(self) -> None: + self.assertIn("python3 tools/live_codex_qualification_harness_v7.py", self.workflow) self.assertIn("qualification_args=(--allow-c13-non-ephemeral-fallback)", self.workflow) self.assertIn("--only C13", self.workflow) self.assertIn("inputs.mode == 'full'", self.workflow) @@ -86,13 +83,13 @@ def test_full_workflow_stays_on_v6_and_enables_baseline23_fallback(self) -> None def test_baseline_and_runbook_remain_23(self) -> None: self.assertIn("Baseline version:** 2.3", self.baseline) self.assertIn("ephemeral-first", self.baseline) - self.assertIn("home-scoped", self.baseline) + self.assertIn("project-scoped non-ephemeral fallback", self.baseline) self.assertIn("baseline 2.3", self.runbook.lower()) - self.assertIn("CODEX_HOME/agents/fixture_agent.toml", self.runbook) + self.assertIn("project-scoped synthetic agent", self.runbook) self.assertIn("project-scoped", self.runbook) def test_safety_boundary_is_not_weakened(self) -> None: - combined = self.source + "\n" + self.compat + "\n" + self.regression + combined = self.source + "\n" + self.v7 + "\n" + self.compat + "\n" + self.regression self.assertNotIn("--dangerously-bypass-approvals-and-sandbox", combined) self.assertNotIn("danger-full-access", combined) self.assertNotIn("--privileged", combined) From 532f99f0f27b204900e022ea137e2ccf9e02d45d Mon Sep 17 00:00:00 2001 From: KeyffMS <124252104+KeyffMS@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:25:40 +0200 Subject: [PATCH 7/9] Align C13 baseline with project fallback --- docs/CODEX_CAPABILITY_BASELINE.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/CODEX_CAPABILITY_BASELINE.md b/docs/CODEX_CAPABILITY_BASELINE.md index 6a74935..2bb4c8f 100644 --- a/docs/CODEX_CAPABILITY_BASELINE.md +++ b/docs/CODEX_CAPABILITY_BASELINE.md @@ -66,7 +66,7 @@ Do not commit session transcripts, credentials, private paths, unrelated Git dat | C10 | `PostCompact` and `SessionStart` can provide recovery context | DOCUMENTED | BLOCKED | Inject only a recovery pointer | | C11 | Project instructions follow documented directory scope and precedence | DOCUMENTED | BLOCKED | Explicitly map affected instructions | | C12 | `project_doc_max_bytes` can truncate automatic instruction loading | DOCUMENTED | BLOCKED | Read, size and hash complete files explicitly | -| C13 | `SubagentStart` can add context but `continue: false` does not stop subagent startup | DOCUMENTED | BLOCKED | Context/audit only; qualify ephemeral-first with a controlled home-scoped fallback when the recognized ephemeral parent-thread blocker occurs | +| C13 | `SubagentStart` can add context but `continue: false` does not stop subagent startup | DOCUMENTED | BLOCKED | Context/audit only; qualify ephemeral-first with a controlled project-scoped non-ephemeral fallback when the recognized ephemeral parent-thread blocker occurs | | C14 | Planning isolation preserves the source branch, SHA, index and files | CONTRACT_DEFINED | BLOCKED | Planning worktree isolation is mandatory | | C15 | Blind review is immutable and detects seeded contract defects | CONTRACT_DEFINED | BLOCKED | Hash review before separate comparison | | C16 | The Git probe accurately reports refs, branches, worktrees, index, commits and cleanup | CONTRACT_DEFINED | BLOCKED | No artifact generation before required Git capabilities pass | @@ -75,7 +75,7 @@ Do not commit session transcripts, credentials, private paths, unrelated Git dat C01, C02, C03 and C05 through C16 must be `REPRODUCED` before production readiness. C04 is informational for PlanAnvil 2.3 because generated execution deliberately forbids nested descendants. -Baseline 2.3 does not mark C13 reproduced from the 2026-09-02 transport diagnostic. That run separated two runtime limitations from the semantic assertion: `codex exec --ephemeral` reproduced the known parent-thread registration failure before `SubagentStart`, while a non-ephemeral attempt progressed past that failure but a project-scoped synthetic custom agent still did not reach `SubagentStart`. The next release-gating run must still obtain real semantic evidence. +Baseline 2.3 does not infer C13 reproduction from transport diagnostics. Controlled Codex 0.152 runs established that `codex exec --ephemeral` can hit a parent-thread registration failure even when project roles are otherwise valid, while non-ephemeral project-agent execution can progress normally. The release-gating fallback must therefore keep both the synthetic role and the `SubagentStart` hook project-scoped and use an isolated user home only for trust, authentication bridging, and disposable persistence. ## 5. Test requirements @@ -89,21 +89,21 @@ Use current documented agent configuration (`agents.enabled` and `agents.max_con ### C13 SubagentStart qualification transport -The semantic assertion under test is the documented `SubagentStart` behavior, not `codex exec --ephemeral` persistence and not project-scoped custom-agent discovery. +The semantic assertion under test is the documented `SubagentStart` behavior, not `codex exec --ephemeral` persistence. C13 therefore uses this fail-closed transport contract: 1. start with a fresh real `codex exec --ephemeral` trial using an aligned project-scoped synthetic agent (`fixture_agent.toml`, declared name `fixture_agent`) and a real project-scoped `SubagentStart` hook; -2. if that trial reaches `SubagentStart`, evaluate the semantics directly and do not use a fallback; -3. permit a non-ephemeral retry only when the ephemeral attempt matches the recognized `collab spawn failed: no thread with id` parent-thread registration failure before `SubagentStart`; -4. for that retry, create a separate disposable repository containing the project-scoped hook/config but no project-scoped custom agent; -5. materialize the synthetic `fixture_agent` only under a private disposable `CODEX_HOME/agents/fixture_agent.toml`, retaining the real project-scoped `SubagentStart` hook as the semantic boundary under test; +2. if that trial reaches `SubagentStart` without the recognized transport failure, evaluate the semantics directly and do not use a fallback; +3. permit a non-ephemeral retry only when the ephemeral attempt matches the recognized `collab spawn failed: no thread with id` parent-thread registration failure; +4. for that retry, create a separate disposable repository that keeps the synthetic `fixture_agent` project-scoped, explicitly declares `[agents.fixture_agent]`, and keeps the real `SubagentStart` hook project-scoped; +5. use a private disposable `CODEX_HOME` only to persist the fixture trust decision, bridge file-backed authentication, and isolate non-ephemeral session/SQLite/log state; do not move the agent or hook into the home layer; 6. keep approval `never`, C13 sandbox `read-only`, model-tool network disabled and project trust limited to the disposable fixture; 7. bridge file-backed authentication only through a temporary symlink, never read or copy the credential file, isolate SQLite/log state, disable message-history persistence, then remove the complete disposable `CODEX_HOME` and verify auth metadata is unchanged; -8. require exactly one real `SubagentStart`, `additionalContext` from that hook, `continue=false` from the same hook, and a child echo of an opaque proof that was not present in the root-agent prompt; +8. require exactly one real project-scoped `SubagentStart`, `additionalContext` from that hook, `continue=false` from the same hook, and a child echo of an opaque proof that was not present in the root-agent prompt; 9. classify missing transport/discovery evidence as `BLOCKED`, and classify contradictory behavior after the real `SubagentStart` boundary is reached as `FAILED`. -The fallback is a qualification transport exception only. It does not make home-scoped custom agents a PlanAnvil product requirement and it does not weaken sandbox, approval, trust, network or evidence-sanitization boundaries. +The fallback is a qualification transport exception only. It does not change PlanAnvil's project-scoped agent/hook product contract and it does not weaken sandbox, approval, trust, network or evidence-sanitization boundaries. ### File handoff @@ -141,7 +141,7 @@ Record separate outcomes for ordinary file writes, temporary refs, branches, lin - Hooks: `https://developers.openai.com/codex/hooks` - AGENTS.md: `https://developers.openai.com/codex/guides/agents-md` -Baseline 2.3 keeps the semantic capability matrix from 2.2 and changes only the C13 qualification transport contract. The change is motivated by controlled live observations plus upstream runtime reports for ephemeral parent-thread registration and project-scoped custom-agent spawning; those reports are diagnostic evidence, not normative sources for the expected `SubagentStart` semantics. +Baseline 2.3 keeps the semantic capability matrix from 2.2 and changes only the C13 qualification transport contract. The transport remains ephemeral-first, but a recognized Codex 0.152 parent-thread registration failure may be retried non-ephemerally without changing the project-scoped role/hook semantics under test. The earlier 2.2 subagent decision remains: current subagent documentation exposes `agents.enabled` and `agents.max_concurrent_threads_per_session`; it does not document `agents.max_depth`. PlanAnvil therefore enforces flat topology in its generated execution contract instead of relying on a runtime depth setting. From 3190e901d084d8d0d41594995abe085dc035aa63 Mon Sep 17 00:00:00 2001 From: KeyffMS <124252104+KeyffMS@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:26:08 +0200 Subject: [PATCH 8/9] Document final Codex 0.152 live transport --- docs/CODEX_SANDBOX_RUNBOOK.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/CODEX_SANDBOX_RUNBOOK.md b/docs/CODEX_SANDBOX_RUNBOOK.md index 6ff091b..12eed86 100644 --- a/docs/CODEX_SANDBOX_RUNBOOK.md +++ b/docs/CODEX_SANDBOX_RUNBOOK.md @@ -29,17 +29,19 @@ The preferred qualification path is `.github/workflows/plananvil-codex-qualifica Use `mode=full` for the release-gating C01-C16 sequence. `mode=c13` remains available as a shorter C13-only probe, but it is not a substitute for the full release gate. -The controlled runner must provide `plananvil-qualification-workspace`. The workflow creates a disposable workspace with that helper, fetches only the exact dispatched `main` SHA, materializes the C01-C16 evidence templates, and runs `tools/live_codex_qualification_harness_v6.py`. Model `gpt-5.6-sol` is pinned, approval policy remains `never`, model-tool network access is disabled, and `workspace-write` is granted only to disposable fixture roots when a trial requires it. Vetted project hooks may bypass only the interactive hook-trust prompt; approval and filesystem sandboxing remain enabled. +The controlled runner must provide `plananvil-qualification-workspace`. The workflow creates a disposable workspace with that helper, fetches only the exact dispatched `main` SHA, materializes the C01-C16 evidence templates, and runs `tools/live_codex_qualification_harness_v7.py`. Model `gpt-5.6-sol` is pinned, approval policy remains `never`, model-tool network access is disabled, and `workspace-write` is granted only to disposable fixture roots when a trial requires it. Vetted project hooks may bypass only the interactive hook-trust prompt; approval and filesystem sandboxing remain enabled. -All normal agent tasks remain ephemeral. Baseline 2.3 introduces exactly one transport exception for C13: the harness may retry C13 non-ephemerally only when the first real ephemeral attempt fails before `SubagentStart` with the recognized `collab spawn failed: no thread with id` parent-thread registration error. Any other ephemeral blocker remains `BLOCKED` and does not activate the exception. +For C08/C09, Codex 0.152 project trust remains a persisted user-config setting, but long full runs must use the runner's real `CODEX_HOME` so the CLI can refresh live authentication normally. The harness temporarily appends only the disposable fixture trust entry to the runner's `config.toml`, removes the invalid CLI trust path, and restores `config.toml` byte-for-byte after the capability. Authentication/session files are not copied or replaced by this trust bridge. + +All normal agent tasks remain ephemeral. Baseline 2.3 introduces exactly one transport exception for C13: the harness may retry C13 non-ephemerally only when the first real ephemeral attempt matches the recognized `collab spawn failed: no thread with id` parent-thread registration error. Any other ephemeral blocker remains `BLOCKED` and does not activate the exception. ### C13 baseline 2.3 transport -C13 tests the documented `SubagentStart` semantics, not persistence of `codex exec --ephemeral` and not project-scoped custom-agent discovery. +C13 tests the documented `SubagentStart` semantics, not persistence of `codex exec --ephemeral`. -The first C13 attempt is still a real project-scoped configuration. The synthetic agent file is `.codex/agents/fixture_agent.toml`, its declared name is `fixture_agent`, the prompt requests `fixture_agent`, and the project-scoped `SubagentStart` matcher targets the same name. The hook injects an outer-generated opaque context proof and intentionally returns `continue=false`. +The first C13 attempt is a real project-scoped configuration. The synthetic agent file is `.codex/agents/fixture_agent.toml`, `[agents.fixture_agent]` points to that file, the prompt requests `fixture_agent`, and the project-scoped `SubagentStart` matcher targets the same name. The hook injects an outer-generated opaque context proof and intentionally returns `continue=false`. -If and only if that ephemeral attempt hits the recognized parent-thread registration failure before the hook, the harness creates a second disposable Git repository. That repository contains the project-scoped C13 hook/config but deliberately contains **no** `.codex/agents` custom agent. The synthetic `fixture_agent` is instead materialized under a private disposable `CODEX_HOME/agents/fixture_agent.toml`. This separates the `SubagentStart` semantic assertion from the independently observed project-scoped custom-agent discovery/spawn limitation while keeping the hook under test project-scoped. +If and only if that ephemeral attempt hits the recognized parent-thread registration failure, the harness creates a second disposable Git repository and retries non-ephemerally. The fallback remains product-aligned: the synthetic agent is still project-scoped, explicitly declared as `[agents.fixture_agent]`, and the real `SubagentStart` hook remains project-scoped. The disposable `CODEX_HOME` is used only for the fixture trust decision, file-backed authentication bridge, and isolated non-ephemeral persistence. The non-ephemeral retry preserves all security boundaries: @@ -55,9 +57,9 @@ The non-ephemeral retry preserves all security boundaries: - the authenticated source `auth.json` metadata must remain unchanged; - evidence retains only structural counts/booleans and never session/thread IDs or the opaque proof value. -C13 is `REPRODUCED` only when exactly one real project-scoped `SubagentStart` hook event occurs, that hook returns both `additionalContext` and `continue=false`, and the real child returns the unseen injected proof. If the semantic boundary is reached but the child lacks the context or startup is stopped by `continue=false`, the result is `FAILED`. If the semantic boundary is not reached or cleanup/auth isolation cannot be proved, the result is `BLOCKED`. +C13 is `REPRODUCED` only when exactly one real project-scoped `SubagentStart` hook event occurs, that hook returns both `additionalContext` and `continue=false`, and the real project-scoped child returns the unseen injected proof. If the semantic boundary is reached but the child lacks the context or startup is stopped by `continue=false`, the result is `FAILED`. If the semantic boundary is not reached or cleanup/auth isolation cannot be proved, the result is `BLOCKED`. -The run #8 diagnostic on Codex 0.152.0 established the reason for this baseline change: ephemeral execution reproduced the parent-thread registration blocker, while a non-ephemeral retry progressed further but the project-scoped synthetic agent still failed before `SubagentStart`. Baseline 2.3 changes the qualification transport only; it does not count that diagnostic as C13 semantic reproduction. +The transport correction is deliberately narrow. It does not move PlanAnvil roles or hooks into user configuration, does not treat a home-scoped synthetic role as product-equivalent, and does not weaken sandbox, approval, trust, network, source-immutability, or evidence-sanitization requirements. ## Evidence and sanitization @@ -82,7 +84,7 @@ python tools/validate_capabilities.py The release-gating controller invocation must include the baseline 2.3 C13 transport permission: ```text -python tools/live_codex_qualification_harness_v6.py \ +python tools/live_codex_qualification_harness_v7.py \ --source-commit \ --run-id \ --output \ From c300035df9420a72d6197e25ed2664c7e3700a80 Mon Sep 17 00:00:00 2001 From: KeyffMS <124252104+KeyffMS@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:32:13 +0200 Subject: [PATCH 9/9] Clarify project-scoped C13 agent wording --- docs/CODEX_SANDBOX_RUNBOOK.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/CODEX_SANDBOX_RUNBOOK.md b/docs/CODEX_SANDBOX_RUNBOOK.md index 12eed86..46eb11e 100644 --- a/docs/CODEX_SANDBOX_RUNBOOK.md +++ b/docs/CODEX_SANDBOX_RUNBOOK.md @@ -39,9 +39,9 @@ All normal agent tasks remain ephemeral. Baseline 2.3 introduces exactly one tra C13 tests the documented `SubagentStart` semantics, not persistence of `codex exec --ephemeral`. -The first C13 attempt is a real project-scoped configuration. The synthetic agent file is `.codex/agents/fixture_agent.toml`, `[agents.fixture_agent]` points to that file, the prompt requests `fixture_agent`, and the project-scoped `SubagentStart` matcher targets the same name. The hook injects an outer-generated opaque context proof and intentionally returns `continue=false`. +The first C13 attempt is a real project-scoped configuration. The project-scoped synthetic agent file is `.codex/agents/fixture_agent.toml`, `[agents.fixture_agent]` points to that file, the prompt requests `fixture_agent`, and the project-scoped `SubagentStart` matcher targets the same name. The hook injects an outer-generated opaque context proof and intentionally returns `continue=false`. -If and only if that ephemeral attempt hits the recognized parent-thread registration failure, the harness creates a second disposable Git repository and retries non-ephemerally. The fallback remains product-aligned: the synthetic agent is still project-scoped, explicitly declared as `[agents.fixture_agent]`, and the real `SubagentStart` hook remains project-scoped. The disposable `CODEX_HOME` is used only for the fixture trust decision, file-backed authentication bridge, and isolated non-ephemeral persistence. +If and only if that ephemeral attempt hits the recognized parent-thread registration failure, the harness creates a second disposable Git repository and retries non-ephemerally. The fallback remains product-aligned: the project-scoped synthetic agent is explicitly declared as `[agents.fixture_agent]`, and the real `SubagentStart` hook remains project-scoped. The disposable `CODEX_HOME` is used only for the fixture trust decision, file-backed authentication bridge, and isolated non-ephemeral persistence. The non-ephemeral retry preserves all security boundaries: