diff --git a/.github/workflows/plananvil-codex-qualification.yml b/.github/workflows/plananvil-codex-qualification.yml index 4cc6887..8ff0f94 100644 --- a/.github/workflows/plananvil-codex-qualification.yml +++ b/.github/workflows/plananvil-codex-qualification.yml @@ -136,12 +136,15 @@ jobs: run: | set -euo pipefail cd "${QUALIFICATION_REPO}" - qualification_args=() + # Baseline 2.3 permits the narrowly gated C13 fallback in both full and + # C13-only modes. The harness still tries ephemeral first and ignores + # this permission unless the recognized parent-thread failure occurs. + qualification_args=(--allow-c13-non-ephemeral-fallback) if [ "${{ inputs.mode }}" = "c13" ]; then - qualification_args+=(--only C13 --allow-c13-non-ephemeral-fallback) + qualification_args+=(--only C13) fi set +e - python3 tools/live_codex_qualification_harness_v5.py \ + python3 tools/live_codex_qualification_harness_v6.py \ --root "${QUALIFICATION_REPO}" \ --source-commit "${GITHUB_SHA}" \ --run-id "${GITHUB_RUN_ID}" \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 856e1c4..d9c459e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ All notable changes to PlanAnvil are documented here. - make C12 a deterministic runtime byte-budget probe with redundant `project_doc_max_bytes` enforcement, secret head/tail markers, zero-tool automatic-loading evidence, and outer PlanAnvil full-file hash verification; - make C06, C08, and C09 deterministic live probes using the real PlanAnvil PreToolUse/PreCompact/PostCompact hooks, explicit postcondition evidence, low-limit `body_after_prefix` auto-compaction triggers, checkpoint repair, repeated compaction, and post-second-compaction continuation checks; - add a diagnostic-only C13 transport mode that keeps `full` strict-ephemeral, retries non-ephemeral only for the recognized parent-thread registration failure, uses an isolated disposable `CODEX_HOME`, requires cleanup/auth-metadata invariants, and verifies real SubagentStart semantics with an outer-generated context proof before any baseline 2.3 decision; +- promote C13 qualification to baseline 2.3: full qualification remains ephemeral-first but may use a known-error-gated non-ephemeral retry with a synthetic home-scoped `fixture_agent` in disposable `CODEX_HOME`, while the real `SubagentStart` hook remains project-scoped and cleanup/auth invariants remain fail-closed; - require production releases to use a GitHub-verified signed annotated tag whose target is reachable from `main`; - fail the production release gate closed when the release worktree is dirty or Git cleanliness cannot be verified; - document the controlled self-hosted Codex qualification path and keep the previous sandbox procedure as a manual fallback. diff --git a/capabilities/C13/README.md b/capabilities/C13/README.md index 28c8a53..7a6161b 100644 --- a/capabilities/C13/README.md +++ b/capabilities/C13/README.md @@ -4,9 +4,11 @@ - Source: `DOCUMENTED` - Release-gating: `yes` - Current result: `BLOCKED` -- Qualification attempt: `2026-08-28` +- Qualification attempt: `2026-09-02` - Documentation check: `PASS` against current Codex hooks documentation. -- Deterministic support: hook code compiles and the full suite passed in run #24. -- Live blocker: no authenticated Codex runtime is available to spawn a subagent and capture the event. +- Baseline: `2.3`. +- Qualification transport: ephemeral-first; only the recognized parent-thread registration failure permits a controlled non-ephemeral retry with a synthetic home-scoped `fixture_agent` in a disposable `CODEX_HOME`, while the real `SubagentStart` hook remains project-scoped. +- Latest diagnostic: run #8 confirmed the ephemeral parent-thread blocker and separately showed that a project-scoped synthetic agent can fail before `SubagentStart`; neither observation counts as semantic reproduction. +- Live blocker: a new full baseline-2.3 run must reach the real `SubagentStart` boundary and verify `additionalContext`, `continue=false`, child context echo, repository immutability, session cleanup, and auth-metadata invariants. Do not change the result to `REPRODUCED` until the complete sanitized live package exists. diff --git a/capabilities/index.json b/capabilities/index.json index 56c6e83..aa9cf29 100644 --- a/capabilities/index.json +++ b/capabilities/index.json @@ -1,5 +1,5 @@ { - "baseline_version": "2.2", + "baseline_version": "2.3", "capabilities": [ {"evidence_directory":"capabilities/C01","expected_behavior":"Repository skills are discovered from .agents/skills","id":"C01","required":true,"result":"BLOCKED","source":"DOCUMENTED"}, {"evidence_directory":"capabilities/C02","expected_behavior":"allow_implicit_invocation false disables implicit invocation while explicit invocation remains available","id":"C02","required":true,"result":"BLOCKED","source":"DOCUMENTED"}, @@ -19,14 +19,13 @@ {"evidence_directory":"capabilities/C16","expected_behavior":"Git probe reports refs, branches, worktrees, index, commits, and cleanup","id":"C16","required":true,"result":"BLOCKED","source":"CONTRACT_DEFINED"} ], "evidence_package_state": "TEMPLATE_ARCHIVE_READY", - "generated_at": "2026-08-28", + "generated_at": "2026-09-02", "qualification_attempt": { - "date": "2026-08-28", - "deterministic_ci_commit": "0e1091fc4168671215ebcd89c92db702ea33ccdd", - "deterministic_ci_run": 26, - "deterministic_ci_result": "PASS", - "live_codex_result": "BLOCKED", - "blocker": "No authenticated Codex runtime or codex executable is available in the qualification execution environment." + "date": "2026-09-02", + "diagnostic_run": 8, + "diagnostic_source_commit": "ebc8cb119434df746b64ffaba3201d0d0ca4c6cd", + "live_codex_result": "PARTIAL", + "blocker": "C13 run #8 separated the ephemeral parent-thread registration blocker and project-scoped synthetic-agent spawn limitation from the SubagentStart semantic assertion; baseline 2.3 prepares the controlled full qualification transport but does not mark C13 REPRODUCED." }, "release_gate": "Required capabilities must have committed sanitized REPRODUCED evidence before production readiness." } diff --git a/docs/CODEX_CAPABILITY_BASELINE.md b/docs/CODEX_CAPABILITY_BASELINE.md index fdaff1e..6a74935 100644 --- a/docs/CODEX_CAPABILITY_BASELINE.md +++ b/docs/CODEX_CAPABILITY_BASELINE.md @@ -1,7 +1,7 @@ # PlanAnvil — Codex Capability Baseline -> **Baseline version:** 2.2 -> **Review date:** 2026-08-28 +> **Baseline version:** 2.3 +> **Review date:** 2026-09-02 > **Purpose:** define current expected Codex behavior and reproducible release tests. > **Authority:** current official OpenAI documentation has precedence. @@ -66,16 +66,16 @@ Do not commit session transcripts, credentials, private paths, unrelated Git dat | C10 | `PostCompact` and `SessionStart` can provide recovery context | DOCUMENTED | BLOCKED | Inject only a recovery pointer | | C11 | Project instructions follow documented directory scope and precedence | DOCUMENTED | BLOCKED | Explicitly map affected instructions | | C12 | `project_doc_max_bytes` can truncate automatic instruction loading | DOCUMENTED | BLOCKED | Read, size and hash complete files explicitly | -| C13 | `SubagentStart` can add context but `continue: false` does not stop subagent startup | DOCUMENTED | BLOCKED | Context and audit only | +| C13 | `SubagentStart` can add context but `continue: false` does not stop subagent startup | DOCUMENTED | BLOCKED | Context/audit only; qualify ephemeral-first with a controlled home-scoped fallback when the recognized ephemeral parent-thread blocker occurs | | C14 | Planning isolation preserves the source branch, SHA, index and files | CONTRACT_DEFINED | BLOCKED | Planning worktree isolation is mandatory | | C15 | Blind review is immutable and detects seeded contract defects | CONTRACT_DEFINED | BLOCKED | Hash review before separate comparison | | C16 | The Git probe accurately reports refs, branches, worktrees, index, commits and cleanup | CONTRACT_DEFINED | BLOCKED | No artifact generation before required Git capabilities pass | ## 4. Release gate -C01, C02, C03 and C05 through C16 must be `REPRODUCED` before production readiness. C04 is informational for PlanAnvil 2.2 because generated execution deliberately forbids nested descendants. +C01, C02, C03 and C05 through C16 must be `REPRODUCED` before production readiness. C04 is informational for PlanAnvil 2.3 because generated execution deliberately forbids nested descendants. -The 2026-08-28 qualification attempt is `BLOCKED` for live Codex reproduction because the available execution environment does not expose an authenticated Codex runtime or `codex` executable. Deterministic contract tests remain useful supporting evidence but cannot substitute for the required live packages. +Baseline 2.3 does not mark C13 reproduced from the 2026-09-02 transport diagnostic. That run separated two runtime limitations from the semantic assertion: `codex exec --ephemeral` reproduced the known parent-thread registration failure before `SubagentStart`, while a non-ephemeral attempt progressed past that failure but a project-scoped synthetic custom agent still did not reach `SubagentStart`. The next release-gating run must still obtain real semantic evidence. ## 5. Test requirements @@ -87,6 +87,24 @@ Verify nested-directory discovery, explicit activation, disabled implicit activa Use current documented agent configuration (`agents.enabled` and `agents.max_concurrent_threads_per_session`). Record the event tree for required reviewer/profiler dispatch. Do not rely on undocumented nesting-depth configuration. Separately assert that generated execution contracts require a flat direct-child topology. +### C13 SubagentStart qualification transport + +The semantic assertion under test is the documented `SubagentStart` behavior, not `codex exec --ephemeral` persistence and not project-scoped custom-agent discovery. + +C13 therefore uses this fail-closed transport contract: + +1. start with a fresh real `codex exec --ephemeral` trial using an aligned project-scoped synthetic agent (`fixture_agent.toml`, declared name `fixture_agent`) and a real project-scoped `SubagentStart` hook; +2. if that trial reaches `SubagentStart`, evaluate the semantics directly and do not use a fallback; +3. permit a non-ephemeral retry only when the ephemeral attempt matches the recognized `collab spawn failed: no thread with id` parent-thread registration failure before `SubagentStart`; +4. for that retry, create a separate disposable repository containing the project-scoped hook/config but no project-scoped custom agent; +5. materialize the synthetic `fixture_agent` only under a private disposable `CODEX_HOME/agents/fixture_agent.toml`, retaining the real project-scoped `SubagentStart` hook as the semantic boundary under test; +6. keep approval `never`, C13 sandbox `read-only`, model-tool network disabled and project trust limited to the disposable fixture; +7. bridge file-backed authentication only through a temporary symlink, never read or copy the credential file, isolate SQLite/log state, disable message-history persistence, then remove the complete disposable `CODEX_HOME` and verify auth metadata is unchanged; +8. require exactly one real `SubagentStart`, `additionalContext` from that hook, `continue=false` from the same hook, and a child echo of an opaque proof that was not present in the root-agent prompt; +9. classify missing transport/discovery evidence as `BLOCKED`, and classify contradictory behavior after the real `SubagentStart` boundary is reached as `FAILED`. + +The fallback is a qualification transport exception only. It does not make home-scoped custom agents a PlanAnvil product requirement and it does not weaken sandbox, approval, trust, network or evidence-sanitization boundaries. + ### File handoff Verify explicit review/profile brief paths, expected hashes and rejection of missing, stale or escaped paths. @@ -115,7 +133,7 @@ Use seeded defects including missing rollback, uncovered requirements, risks wit Record separate outcomes for ordinary file writes, temporary refs, branches, linked worktrees, index updates, commits, signing, repository hooks and cleanup under every supported permission mode. -## 6. Current official documentation checked on 2026-08-28 +## 6. Current official documentation checked on 2026-09-02 - Skills: `https://developers.openai.com/codex/skills` - Configuration reference: `https://developers.openai.com/codex/config-reference` @@ -123,7 +141,9 @@ Record separate outcomes for ordinary file writes, temporary refs, branches, lin - Hooks: `https://developers.openai.com/codex/hooks` - AGENTS.md: `https://developers.openai.com/codex/guides/agents-md` -Notable drift from baseline 2.1: current subagent documentation exposes `agents.enabled` and `agents.max_concurrent_threads_per_session`; it does not document `agents.max_depth`. `agents.max_threads` is retained only as a legacy concurrency alias. PlanAnvil therefore enforces flat topology in its generated execution contract instead of relying on a runtime depth setting. +Baseline 2.3 keeps the semantic capability matrix from 2.2 and changes only the C13 qualification transport contract. The change is motivated by controlled live observations plus upstream runtime reports for ephemeral parent-thread registration and project-scoped custom-agent spawning; those reports are diagnostic evidence, not normative sources for the expected `SubagentStart` semantics. + +The earlier 2.2 subagent decision remains: current subagent documentation exposes `agents.enabled` and `agents.max_concurrent_threads_per_session`; it does not document `agents.max_depth`. PlanAnvil therefore enforces flat topology in its generated execution contract instead of relying on a runtime depth setting. ## 7. Architecture rule diff --git a/docs/CODEX_SANDBOX_RUNBOOK.md b/docs/CODEX_SANDBOX_RUNBOOK.md index 1924b63..6ff091b 100644 --- a/docs/CODEX_SANDBOX_RUNBOOK.md +++ b/docs/CODEX_SANDBOX_RUNBOOK.md @@ -27,19 +27,47 @@ If the probe fails with `setting up uid map: Operation not permitted`, fix the P The preferred qualification path is `.github/workflows/plananvil-codex-qualification.yml`. The workflow is intentionally `workflow_dispatch`-only, accepts execution only from `main`, uses Environment `plananvil-codex`, and targets `[self-hosted, linux, x64, plananvil, codex]`. -Use `mode=full` for the release-gating C01-C16 sequence. While issue #17 is being resolved, `mode=c13` is a **diagnostic-only** transport comparison for C13; it does not change baseline 2.2 and it is not a substitute for the full release gate. +Use `mode=full` for the release-gating C01-C16 sequence. `mode=c13` remains available as a shorter C13-only probe, but it is not a substitute for the full release gate. -The controlled runner must provide `plananvil-qualification-workspace`. The workflow creates a disposable workspace with that helper, fetches only the exact dispatched `main` SHA, materializes the C01-C16 evidence templates, and runs the current live qualification harness. In `full` mode the qualification contract remains strict-ephemeral: agent tasks use `codex exec --ephemeral`, model `gpt-5.6-sol`, approval policy `never`, model-tool network disabled, and `workspace-write` only for disposable fixture roots that require writes. Vetted project hooks may bypass the interactive hook-trust prompt; approval and filesystem sandboxing remain enabled. +The controlled runner must provide `plananvil-qualification-workspace`. The workflow creates a disposable workspace with that helper, fetches only the exact dispatched `main` SHA, materializes the C01-C16 evidence templates, and runs `tools/live_codex_qualification_harness_v6.py`. Model `gpt-5.6-sol` is pinned, approval policy remains `never`, model-tool network access is disabled, and `workspace-write` is granted only to disposable fixture roots when a trial requires it. Vetted project hooks may bypass only the interactive hook-trust prompt; approval and filesystem sandboxing remain enabled. -The C13 diagnostic first runs the same real project-scoped SubagentStart fixture through `codex exec --ephemeral`. A non-ephemeral retry is permitted only when the ephemeral attempt matches the recognized `collab spawn failed: no thread with id` parent-thread registration failure. The retry keeps C13 read-only, approval `never`, model-tool network disabled, the trusted disposable project, and the real SubagentStart hook/child agent. +All normal agent tasks remain ephemeral. Baseline 2.3 introduces exactly one transport exception for C13: the harness may retry C13 non-ephemerally only when the first real ephemeral attempt fails before `SubagentStart` with the recognized `collab spawn failed: no thread with id` parent-thread registration error. Any other ephemeral blocker remains `BLOCKED` and does not activate the exception. -For that diagnostic retry the harness creates a separate disposable `CODEX_HOME`. It does not read or copy the authenticated runner's `auth.json`; when file-backed authentication is available it creates only a temporary symlink to the existing auth file, redirects SQLite/log storage into the disposable home, disables message-history persistence, records only the number of newly created rollout files, verifies the auth file metadata is unchanged, and removes the entire disposable home before accepting the trial. If file-backed authentication is unavailable, session persistence cannot be proven, cleanup fails, auth metadata changes, or any command/file tool is used by the root C13 probe, the diagnostic fails closed as `BLOCKED`. +### C13 baseline 2.3 transport -The C13 context proof is generated by the outer harness and is never included in the root-agent prompt or retained in evidence. A real SubagentStart proxy records only structural facts (`continue=false`, presence of `additionalContext`, event count) while delegating to the real hook. `REPRODUCED` requires exactly one real SubagentStart event and a child echo of the unseen injected proof. This diagnostic evidence is intended to decide issue #17; enabling the non-ephemeral fallback in `full` mode requires a separate baseline 2.3 contract update. +C13 tests the documented `SubagentStart` semantics, not persistence of `codex exec --ephemeral` and not project-scoped custom-agent discovery. -Raw Codex session streams are not retained. The controller keeps only sanitized final assertions, event-type counts, and relative Git structure required for evaluation. The self-hosted runner has repository read permission only and never pushes qualification changes. +The first C13 attempt is still a real project-scoped configuration. The synthetic agent file is `.codex/agents/fixture_agent.toml`, its declared name is `fixture_agent`, the prompt requests `fixture_agent`, and the project-scoped `SubagentStart` matcher targets the same name. The hook injects an outer-generated opaque context proof and intentionally returns `continue=false`. -The workflow performs the Linux `bubblewrap` user-namespace probe before preparing fixtures, so an incompatible container fails in seconds instead of consuming a live qualification run. It uploads `plananvil-codex-evidence-` as a short-lived artifact. Review that artifact before committing evidence through a normal protected pull request. A `full` workflow run exits successfully only when every release-gating capability is `REPRODUCED`; a `c13` diagnostic exits successfully only when C13 semantics are reproduced under the controlled transport comparison. Partial/failed runs still upload their sanitized evidence artifact for diagnosis. +If and only if that ephemeral attempt hits the recognized parent-thread registration failure before the hook, the harness creates a second disposable Git repository. That repository contains the project-scoped C13 hook/config but deliberately contains **no** `.codex/agents` custom agent. The synthetic `fixture_agent` is instead materialized under a private disposable `CODEX_HOME/agents/fixture_agent.toml`. This separates the `SubagentStart` semantic assertion from the independently observed project-scoped custom-agent discovery/spawn limitation while keeping the hook under test project-scoped. + +The non-ephemeral retry preserves all security boundaries: + +- C13 remains `read-only`; +- approval remains `never`; +- model-tool network access remains disabled; +- the Git repository is disposable and explicitly trusted only for the trial; +- the root C13 session may not use command/file mutation tools; +- file-backed authentication is bridged only by a temporary symlink to the existing `auth.json`; the harness does not read or copy its contents; +- SQLite and log paths are redirected into the disposable `CODEX_HOME`; +- message history persistence is disabled; +- the entire disposable `CODEX_HOME` is removed after the retry; +- the authenticated source `auth.json` metadata must remain unchanged; +- evidence retains only structural counts/booleans and never session/thread IDs or the opaque proof value. + +C13 is `REPRODUCED` only when exactly one real project-scoped `SubagentStart` hook event occurs, that hook returns both `additionalContext` and `continue=false`, and the real child returns the unseen injected proof. If the semantic boundary is reached but the child lacks the context or startup is stopped by `continue=false`, the result is `FAILED`. If the semantic boundary is not reached or cleanup/auth isolation cannot be proved, the result is `BLOCKED`. + +The run #8 diagnostic on Codex 0.152.0 established the reason for this baseline change: ephemeral execution reproduced the parent-thread registration blocker, while a non-ephemeral retry progressed further but the project-scoped synthetic agent still failed before `SubagentStart`. Baseline 2.3 changes the qualification transport only; it does not count that diagnostic as C13 semantic reproduction. + +## Evidence and sanitization + +Raw Codex session streams are not retained. The controller keeps only sanitized final assertions, event-type counts, hashes and relative Git structure required for evaluation. The self-hosted runner has repository read permission only and never pushes qualification changes. + +The workflow performs the Linux `bubblewrap` user-namespace probe before preparing fixtures, so an incompatible container fails in seconds instead of consuming a live qualification run. It uploads `plananvil-codex-evidence-` as a short-lived artifact. Review that artifact before committing evidence through a normal protected pull request. A `full` workflow run exits successfully only when every release-gating capability is `REPRODUCED`; partial/failed runs still upload their sanitized evidence artifact for diagnosis. + +Do not commit session transcripts. Retain the minimum event/decision structure needed to evaluate assertions. Replace usernames, home directories, temporary absolute paths, repository URLs, IDs, credentials and session identifiers with stable placeholders or hashes. + +The validator rejects obvious token/private-path patterns in `actual.sanitized.json`. ## Manual fallback @@ -51,15 +79,27 @@ python tools/prepare_capabilities.py --force python tools/validate_capabilities.py ``` +The release-gating controller invocation must include the baseline 2.3 C13 transport permission: + +```text +python tools/live_codex_qualification_harness_v6.py \ + --source-commit \ + --run-id \ + --output \ + --allow-c13-non-ephemeral-fallback +``` + +The flag is only permission for the narrow fallback. It does not force non-ephemeral execution; C13 always attempts ephemeral execution first. + ## Per-capability sequence For C01 through C16 in order: 1. read `capabilities/CXX/README.md`; -2. instantiate the files described by `fixture/README.md` in a disposable Git repository; -3. apply the capability-specific configuration described in `config/README.md`; -4. execute the prompt in `prompt.txt` using the invocation recorded in `run-command.txt`; -5. capture only the structural observations required by `expected.json`; +2. instantiate the files described by the prepared capability package in a disposable Git repository; +3. apply the capability-specific deterministic harness setup where the current controller defines one; +4. execute the live trial using the least-privilege sandbox required by that capability; +5. capture only the structural observations required by the capability assertion; 6. sanitize the result into `actual.sanitized.json`; 7. set `evaluation.json` to `REPRODUCED`, `FAILED`, or `BLOCKED` based on the assertions; 8. when `REPRODUCED`, fill exact runtime metadata (`codex_version`, `model`, `os`, `permission_mode`, `project_trust`); @@ -67,13 +107,7 @@ For C01 through C16 in order: 10. run `python tools/rehash_capability.py CXX`; 11. run `python tools/validate_capabilities.py` before continuing. -C04 is informational/non-gating in baseline 2.2 but should still be observed if the sandbox exposes nested-subagent behavior relevant to future architecture. - -## Sanitization - -Do not commit session transcripts. Retain the minimum event/decision structure needed to evaluate assertions. Replace usernames, home directories, temporary absolute paths, repository URLs, IDs, credentials, and session identifiers with stable placeholders or hashes. - -The validator rejects obvious token/private-path patterns in `actual.sanitized.json`. +C04 is informational/non-gating in baseline 2.3 but should still be observed if the sandbox exposes nested-subagent behavior relevant to future architecture. ## Final gate diff --git a/tests/test_live_codex_qualification_harness_v4.py b/tests/test_live_codex_qualification_harness_v4.py index f2480ed..f8daec5 100644 --- a/tests/test_live_codex_qualification_harness_v4.py +++ b/tests/test_live_codex_qualification_harness_v4.py @@ -55,12 +55,16 @@ def test_safety_boundary_is_not_weakened(self) -> None: self.assertIn('sandbox="read-only"', self.source) self.assertIn('sandbox="workspace-write"', self.source) - def test_v4_is_chained_under_current_v5_wrapper(self) -> None: - current = (ROOT / "tools" / "live_codex_qualification_harness_v5.py").read_text( + def test_v4_is_chained_under_current_v6_wrapper(self) -> None: + v5 = (ROOT / "tools" / "live_codex_qualification_harness_v5.py").read_text( encoding="utf-8" ) - self.assertIn("import live_codex_qualification_harness_v4 as prior", current) - self.assertIn("python3 tools/live_codex_qualification_harness_v5.py", self.workflow) + v6 = (ROOT / "tools" / "live_codex_qualification_harness_v6.py").read_text( + encoding="utf-8" + ) + self.assertIn("import live_codex_qualification_harness_v4 as prior", v5) + self.assertIn("import live_codex_qualification_harness_v5 as prior", v6) + self.assertIn("python3 tools/live_codex_qualification_harness_v6.py", self.workflow) if __name__ == "__main__": diff --git a/tests/test_live_codex_qualification_harness_v5.py b/tests/test_live_codex_qualification_harness_v5.py index 4baa08a..bbf54ab 100644 --- a/tests/test_live_codex_qualification_harness_v5.py +++ b/tests/test_live_codex_qualification_harness_v5.py @@ -54,7 +54,7 @@ def test_non_ephemeral_home_is_isolated_without_copying_auth(self) -> None: self.assertIn("session_cleanup_verified", self.source) self.assertIn("auth_metadata_unchanged", self.source) - def test_full_release_path_cannot_enable_fallback_yet(self) -> None: + def test_v5_historical_controller_remains_diagnostic_only(self) -> None: self.assertIn( "C13 non-ephemeral fallback is diagnostic-only until the baseline contract is updated", self.source, @@ -72,19 +72,20 @@ def test_safety_boundary_is_not_weakened(self) -> None: self.assertIn("completed_file_change_items", self.source) self.assertIn("repository_unchanged", self.source) - def test_workflow_exposes_c13_diagnostic_but_keeps_full_separate(self) -> None: + def test_current_workflow_uses_v6_and_keeps_c13_short_mode(self) -> None: self.assertIn("- c13", self.workflow) self.assertIn("inputs.mode == 'c13'", self.workflow) - self.assertIn("python3 tools/live_codex_qualification_harness_v5.py", self.workflow) + self.assertIn("python3 tools/live_codex_qualification_harness_v6.py", self.workflow) self.assertIn("--only C13", self.workflow) self.assertIn("--allow-c13-non-ephemeral-fallback", self.workflow) self.assertIn("inputs.mode == 'full'", self.workflow) - def test_runbook_marks_c13_mode_diagnostic_only(self) -> None: - self.assertIn("`c13`", self.runbook) - self.assertIn("diagnostic", self.runbook.lower()) - self.assertIn("baseline 2.3", self.runbook) - self.assertIn("full", self.runbook) + def test_runbook_documents_baseline23_full_transport(self) -> None: + self.assertIn("mode=c13", self.runbook) + self.assertIn("baseline 2.3", self.runbook.lower()) + self.assertIn("CODEX_HOME/agents/fixture_agent.toml", self.runbook) + self.assertIn("project-scoped", self.runbook) + self.assertIn("mode=full", self.runbook) if __name__ == "__main__": diff --git a/tests/test_live_codex_qualification_harness_v6.py b/tests/test_live_codex_qualification_harness_v6.py new file mode 100644 index 0000000..4afbf9f --- /dev/null +++ b/tests/test_live_codex_qualification_harness_v6.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = ROOT / "tools" / "live_codex_qualification_harness_v6.py" +WORKFLOW_PATH = ROOT / ".github" / "workflows" / "plananvil-codex-qualification.yml" +BASELINE_PATH = ROOT / "docs" / "CODEX_CAPABILITY_BASELINE.md" +RUNBOOK_PATH = ROOT / "docs" / "CODEX_SANDBOX_RUNBOOK.md" + + +class LiveCodexHarnessV6Tests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.source = MODULE_PATH.read_text(encoding="utf-8") + cls.workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + cls.baseline = BASELINE_PATH.read_text(encoding="utf-8") + cls.runbook = RUNBOOK_PATH.read_text(encoding="utf-8") + + def test_v6_overrides_only_c13_and_inherits_v5(self) -> None: + self.assertIn('TARGET_CAPABILITIES = {"C13"}', self.source) + self.assertIn("import live_codex_qualification_harness_v5 as prior", self.source) + self.assertIn("_ORIGINAL_CAPABILITY_RUNTIME = prior.capability_runtime", self.source) + + def test_agent_identity_is_aligned(self) -> None: + self.assertIn('HOME_AGENT_NAME = "fixture_agent"', self.source) + self.assertIn('HOME_AGENT_FILENAME = "fixture_agent.toml"', self.source) + self.assertIn('"matcher": f"^{HOME_AGENT_NAME}$"', self.source) + self.assertIn("agent_name_matches_filename", self.source) + + def test_ephemeral_attempt_remains_project_scoped(self) -> None: + self.assertIn("_seed_project_fixture(project_repo, proof, include_project_agent=True)", self.source) + self.assertIn("ephemeral=True", self.source) + self.assertIn('trial_e["agent_fixture_scope"] = "project"', self.source) + + def test_fallback_separates_agent_discovery_from_project_hook(self) -> None: + self.assertIn("_seed_project_fixture(fallback_repo, proof, include_project_agent=False)", self.source) + self.assertIn('home / "agents" / HOME_AGENT_FILENAME', self.source) + self.assertIn('trial_n["agent_fixture_scope"] = "disposable_CODEX_HOME"', self.source) + self.assertIn('trial_n["project_agent_present"] = False', self.source) + self.assertIn('trial_n["project_scoped_subagent_start_hook"] = True', self.source) + + def test_fallback_is_still_known_error_gated(self) -> None: + self.assertIn("known_e and ALLOW_NON_EPHEMERAL_FALLBACK", self.source) + self.assertIn("ephemeral_known_transport_blocker_fallback_not_enabled", self.source) + self.assertIn("non_ephemeral_home_agent_fallback", self.source) + + def test_non_ephemeral_cleanup_and_auth_invariants_remain_required(self) -> None: + self.assertIn("prior._prepare_isolated_codex_home", self.source) + self.assertIn("prior._cleanup_isolated_codex_home", self.source) + self.assertIn("session_cleanup_verified", self.source) + self.assertIn("auth_metadata_unchanged", self.source) + self.assertIn("home_scoped_fixture_agent_materialized", self.source) + + def test_full_workflow_enables_baseline23_fallback(self) -> None: + self.assertIn("python3 tools/live_codex_qualification_harness_v6.py", self.workflow) + self.assertIn("qualification_args=(--allow-c13-non-ephemeral-fallback)", self.workflow) + self.assertIn("--only C13", self.workflow) + self.assertIn("inputs.mode == 'full'", self.workflow) + + def test_baseline_and_runbook_are_23(self) -> None: + self.assertIn("Baseline version:** 2.3", self.baseline) + self.assertIn("ephemeral-first", self.baseline) + self.assertIn("home-scoped", self.baseline) + self.assertIn("baseline 2.3", self.runbook.lower()) + self.assertIn("CODEX_HOME/agents/fixture_agent.toml", self.runbook) + self.assertIn("project-scoped", self.runbook) + + def test_safety_boundary_is_not_weakened(self) -> None: + self.assertNotIn("--dangerously-bypass-approvals-and-sandbox", self.source) + self.assertNotIn("danger-full-access", self.source) + self.assertNotIn("--privileged", self.source) + self.assertNotIn("SYS_ADMIN", self.source) + self.assertIn('sandbox_mode = "read-only"', self.source) + self.assertIn("base.git_snapshot", self.source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_prepare_capabilities_overlay.py b/tests/test_prepare_capabilities_overlay.py new file mode 100644 index 0000000..8d24c4c --- /dev/null +++ b/tests/test_prepare_capabilities_overlay.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import json +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) + +import prepare_capabilities +import validate_capabilities + + +class CapabilityMaterializerOverlayTests(unittest.TestCase): + def test_c13_baseline23_overlay_materializes_and_rehashes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) / "materialized" + written = prepare_capabilities.materialize(ROOT, target, force=True) + + c13 = target / "capabilities" / "C13" + self.assertIn("capabilities/C13/hashes.json", written) + self.assertIn("Baseline: `2.3`", (c13 / "README.md").read_text(encoding="utf-8")) + self.assertIn( + "live_codex_qualification_harness_v6.py", + (c13 / "run-command.txt").read_text(encoding="utf-8"), + ) + config = (c13 / "config" / "README.md").read_text(encoding="utf-8") + self.assertIn("home-scoped", config) + self.assertIn("project-scoped", config) + self.assertIn("fixture_agent.toml", config) + + expected = json.loads((c13 / "expected.json").read_text(encoding="utf-8")) + self.assertEqual( + expected["assertions"], + [ + "SubagentStart can add context for the starting agent.", + "continue=false does not become a relied-upon startup blocker.", + ], + ) + index = json.loads((target / "capabilities" / "index.json").read_text(encoding="utf-8")) + self.assertEqual(index["baseline_version"], "2.3") + self.assertEqual(validate_capabilities.validate_all(target), []) + + def test_overlay_does_not_remove_other_capabilities(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) / "materialized" + prepare_capabilities.materialize(ROOT, target, force=True) + for capability_id in ("C01", "C06", "C12", "C16"): + self.assertTrue((target / "capabilities" / capability_id / "expected.json").is_file()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/live_codex_qualification_harness_v6.py b/tools/live_codex_qualification_harness_v6.py new file mode 100644 index 0000000..2d69cbb --- /dev/null +++ b/tools/live_codex_qualification_harness_v6.py @@ -0,0 +1,381 @@ +from __future__ import annotations + +import json +import shutil +import sys +from pathlib import Path +from typing import Any + +import live_codex_qualification_harness_v5 as prior + +base = prior.base + +TARGET_CAPABILITIES = {"C13"} +_ORIGINAL_CAPABILITY_RUNTIME = prior.capability_runtime +ALLOW_NON_EPHEMERAL_FALLBACK = False +HOME_AGENT_NAME = "fixture_agent" +HOME_AGENT_FILENAME = "fixture_agent.toml" + + +def _write(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _agent_toml() -> str: + return ( + f'name = "{HOME_AGENT_NAME}"\n' + 'description = "C13 qualification child for real SubagentStart context semantics."\n' + 'sandbox_mode = "read-only"\n' + 'developer_instructions = """\n' + 'Do not use tools. Inspect only your already-active startup/developer context.\n' + f'If it contains a line beginning `{prior.C13_CONTEXT_PREFIX}`, reply exactly\n' + f'`{prior.C13_ECHO_PREFIX}` where is the exact opaque value after the equals sign.\n' + f'If no such line is present, reply exactly `{prior.C13_MISSING}`. Never guess a value.\n' + '"""\n' + ) + + +def _seed_project_fixture(repo: Path, context_proof: str, *, include_project_agent: bool) -> None: + _write( + repo / ".codex" / "config.toml", + """[agents] +enabled = true +max_concurrent_threads_per_session = 2 +""", + ) + if include_project_agent: + _write(repo / ".codex" / "agents" / HOME_AGENT_FILENAME, _agent_toml()) + + _write( + repo / ".codex" / "hooks" / "subagent-start-fixture.py", + "from __future__ import annotations\n" + "import json\n\n" + "print(json.dumps({\n" + ' "continue": False,\n' + ' "stopReason": "C13 compatibility signal: continue=false must not block SubagentStart",\n' + ' "hookSpecificOutput": {\n' + ' "hookEventName": "SubagentStart",\n' + f' "additionalContext": "{prior.C13_CONTEXT_PREFIX}{context_proof}",\n' + " },\n" + "}, sort_keys=True))\n", + ) + _write( + repo / ".codex" / "hooks" / "qualification-c13-hook-proxy.py", + prior._c13_hook_proxy_source(), + ) + _write( + repo / ".codex" / "hooks.json", + json.dumps( + { + "hooks": { + "SubagentStart": [ + { + "matcher": f"^{HOME_AGENT_NAME}$", + "hooks": [ + { + "type": "command", + "command": ( + 'python3 "$(git rev-parse --show-toplevel)/.codex/hooks/' + 'qualification-c13-hook-proxy.py"' + ), + "timeout": 30, + } + ], + } + ] + } + }, + indent=2, + sort_keys=True, + ) + + "\n", + ) + scope = "project-agent" if include_project_agent else "project-hook-only" + _write(repo / "README.md", f"C13 deterministic {scope} qualification fixture.\n") + gitignore = repo / ".gitignore" + existing = gitignore.read_text(encoding="utf-8") if gitignore.exists() else "" + if prior.C13_HOOK_LOG_RELATIVE not in existing: + with gitignore.open("a", encoding="utf-8") as handle: + if existing and not existing.endswith("\n"): + handle.write("\n") + handle.write(prior.C13_HOOK_LOG_RELATIVE + "\n") + + +def _prepare_home_scoped_fallback_agent( + cap_runtime: Path, +) -> tuple[Path, Path | None, tuple[int, int, int] | None, bool]: + home, auth_path, auth_before = prior._prepare_isolated_codex_home(cap_runtime) + agent_path = home / "agents" / HOME_AGENT_FILENAME + _write(agent_path, _agent_toml()) + return home, auth_path, auth_before, agent_path.is_file() + + +def _c13_runtime( + *, + root: Path, + runtime_root: Path, + schemas: dict[str, Path], + version: str, + os_name: str, + source_commit: str, + date: str, +) -> tuple[str, bool]: + capability_id = "C13" + ( + _cap_dir, + cap_runtime, + _spec_dir, + project_repo, + _worktrees, + results_dir, + _evaluator_dir, + ) = prior._runtime_paths(root=root, runtime_root=runtime_root, capability_id=capability_id) + fallback_repo = cap_runtime / "fallback-repo" + + with prior.prior.v2._python_bytecode_disabled(): + proof = prior._context_proof(source_commit) + + # Attempt 1 remains project-scoped and ephemeral. Filename, declared name, + # prompt target, and hook matcher are deliberately identical. + base.ensure_git_repo(project_repo) + _seed_project_fixture(project_repo, proof, include_project_agent=True) + project_fixture_commit = base.commit_fixture_baseline(project_repo) + + prior._clear_hook_log(project_repo) + before_ephemeral = base.git_snapshot(project_repo) + payload_e, events_e, error_e, known_e = prior._run_c13_codex( + repo=project_repo, + schemas=schemas, + results_dir=results_dir, + position=1, + ephemeral=True, + timeout=600, + ) + after_ephemeral = base.git_snapshot(project_repo) + records_e = prior._read_hook_records(project_repo) + outcome_e, trial_e = prior._evaluate_transport( + transport="ephemeral", + payload=payload_e, + events=events_e, + error=error_e, + known_parent_failure=known_e, + records=records_e, + context_proof=proof, + git_before=before_ephemeral, + git_after=after_ephemeral, + ) + trial_e["agent_fixture_scope"] = "project" + trial_e["agent_name_matches_filename"] = True + + trials: list[dict[str, Any]] = [base.sanitize(trial_e)] + fallback_used = False + fallback_available = False + cleanup_verified = True + auth_unchanged = True + home_agent_materialized = False + fallback_fixture_commit: str | None = None + + if outcome_e == "PASS": + final_outcome = "PASS" + transport_resolution = "ephemeral" + elif outcome_e == "FAILED": + final_outcome = "FAILED" + transport_resolution = "ephemeral_semantic_failure" + elif not (known_e and ALLOW_NON_EPHEMERAL_FALLBACK): + final_outcome = "BLOCKED" + transport_resolution = ( + "ephemeral_known_transport_blocker_fallback_not_enabled" + if known_e + else "ephemeral_unclassified_blocker" + ) + else: + # Run #8 proved that non-ephemeral registration survives the parent-thread + # failure but the project-scoped synthetic agent can still be rejected before + # SubagentStart. Baseline 2.3 therefore isolates the semantic probe from that + # independent discovery limitation: the synthetic child is home-scoped inside + # a disposable CODEX_HOME while the hook under test remains project-scoped. + fallback_used = True + base.ensure_git_repo(fallback_repo) + _seed_project_fixture(fallback_repo, proof, include_project_agent=False) + fallback_fixture_commit = base.commit_fixture_baseline(fallback_repo) + + isolated_home, auth_path, auth_before, home_agent_materialized = ( + _prepare_home_scoped_fallback_agent(cap_runtime) + ) + fallback_available = auth_path is not None and home_agent_materialized + + if not fallback_available: + shutil.rmtree(isolated_home, ignore_errors=True) + cleanup_verified = not isolated_home.exists() + final_outcome = "BLOCKED" + transport_resolution = "isolated_non_ephemeral_fallback_preflight_unavailable" + trials.append( + { + "capability_id": capability_id, + "trial": "non_ephemeral_home_agent_fallback_preflight", + "trial_name": "non_ephemeral_home_agent_fallback_preflight", + "transport": "non-ephemeral", + "outcome": "BLOCKED", + "assertions": [], + "observations": [ + "home_scoped_fixture_agent_materialized=" + f"{str(home_agent_materialized).lower()}", + "file_backed_auth_bridge_available=" + f"{str(auth_path is not None).lower()}", + f"session_cleanup_verified={str(cleanup_verified).lower()}", + ], + "blocker": "isolated non-ephemeral home-agent fallback preflight unavailable", + } + ) + else: + prior._clear_hook_log(fallback_repo) + before_fallback = base.git_snapshot(fallback_repo) + payload_n: dict[str, Any] = {} + events_n: dict[str, Any] = {} + error_n: str | None = None + known_n = False + session_rollouts = 0 + after_fallback = before_fallback + records_n: list[dict[str, Any]] = [] + try: + payload_n, events_n, error_n, known_n = prior._run_c13_codex( + repo=fallback_repo, + schemas=schemas, + results_dir=results_dir, + position=2, + ephemeral=False, + isolated_codex_home=isolated_home, + timeout=600, + ) + session_rollouts = prior._session_rollout_count(isolated_home) + after_fallback = base.git_snapshot(fallback_repo) + records_n = prior._read_hook_records(fallback_repo) + finally: + cleanup_verified, auth_unchanged = prior._cleanup_isolated_codex_home( + isolated_home, + auth_path, + auth_before, + ) + + outcome_n, trial_n = prior._evaluate_transport( + transport="non-ephemeral", + payload=payload_n, + events=events_n, + error=error_n, + known_parent_failure=known_n, + records=records_n, + context_proof=proof, + git_before=before_fallback, + git_after=after_fallback, + session_rollouts_created=session_rollouts, + session_cleanup_verified=cleanup_verified, + auth_unchanged=auth_unchanged, + ) + trial_n["agent_fixture_scope"] = "disposable_CODEX_HOME" + trial_n["project_agent_present"] = False + trial_n["home_agent_materialized"] = home_agent_materialized + trial_n["project_scoped_subagent_start_hook"] = True + trial_n["agent_name_matches_filename"] = True + trial_n["fallback_fixture_commit"] = fallback_fixture_commit + trials.append(base.sanitize(trial_n)) + final_outcome = outcome_n + transport_resolution = "non_ephemeral_home_agent_fallback" + + if final_outcome == "PASS": + result = "REPRODUCED" + expected_met = True + blocker = None + if transport_resolution == "ephemeral": + summary = ( + "C13 reproduced directly under ephemeral execution with one real " + "SubagentStart hook and child context echo." + ) + else: + summary = ( + "C13 reproduced through the baseline 2.3 controlled non-ephemeral " + "home-agent fallback after the recognized ephemeral parent-thread " + "registration failure; the SubagentStart hook remained project-scoped." + ) + elif final_outcome == "FAILED": + result = "FAILED" + expected_met = False + blocker = ( + "C13 reached real SubagentStart startup but observed semantics contradicted " + "the expected context/continue=false contract." + ) + summary = "C13 failed after reaching the real SubagentStart semantic boundary." + else: + result = "BLOCKED" + expected_met = False + blocker = ( + "C13 could not reach and verify the real child startup semantics under the " + "baseline 2.3 permitted qualification transport." + ) + summary = ( + "C13 remains blocked because the real SubagentStart semantic boundary was " + "not completely exercised." + ) + + evidence_fixture_commit = ( + fallback_fixture_commit if fallback_used and fallback_fixture_commit else project_fixture_commit + ) + return prior.prior._write_result( + root=root, + cap_runtime=cap_runtime, + capability_id=capability_id, + result=result, + expected_met=expected_met, + observations=[ + f"ephemeral_outcome={outcome_e}", + f"ephemeral_known_parent_thread_failure={str(known_e).lower()}", + f"non_ephemeral_fallback_enabled={str(ALLOW_NON_EPHEMERAL_FALLBACK).lower()}", + f"non_ephemeral_fallback_used={str(fallback_used).lower()}", + f"non_ephemeral_fallback_available={str(fallback_available).lower()}", + f"home_scoped_fixture_agent_materialized={str(home_agent_materialized).lower()}", + "fallback_project_agent_present=false" if fallback_used else "fallback_not_used=true", + f"session_cleanup_verified={str(cleanup_verified).lower()}", + f"auth_metadata_unchanged={str(auth_unchanged).lower()}", + f"transport_resolution={transport_resolution}", + ], + blocker=blocker, + summary=summary, + trials=trials, + fixture_commit=evidence_fixture_commit, + version=version, + os_name=os_name, + source_commit=source_commit, + date=date, + ) + + +def capability_runtime(**kwargs: Any) -> tuple[str, bool]: + capability_id = str(kwargs["capability_id"]) + if capability_id not in TARGET_CAPABILITIES: + return _ORIGINAL_CAPABILITY_RUNTIME(**kwargs) + common = {key: value for key, value in kwargs.items() if key != "capability_id"} + return _c13_runtime(**common) + + +def main(argv: list[str] | None = None) -> int: + global ALLOW_NON_EPHEMERAL_FALLBACK + args = list(sys.argv[1:] if argv is None else argv) + allow_flag = "--allow-c13-non-ephemeral-fallback" + ALLOW_NON_EPHEMERAL_FALLBACK = allow_flag in args + args = [item for item in args if item != allow_flag] + + # The v5 C13-only controller is still useful as a short diagnostic surface, + # but baseline 2.3 also permits the same narrowly gated fallback in full mode. + base.capability_runtime = capability_runtime + prior.capability_runtime = capability_runtime + if "--only" in args: + if not ALLOW_NON_EPHEMERAL_FALLBACK: + raise base.QualificationError( + "C13-only qualification requires --allow-c13-non-ephemeral-fallback" + ) + return prior._c13_only_main(args) + return base.main(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/prepare_capabilities.py b/tools/prepare_capabilities.py index 3e29402..bb91bec 100644 --- a/tools/prepare_capabilities.py +++ b/tools/prepare_capabilities.py @@ -2,13 +2,103 @@ import argparse import base64 +import hashlib import io +import json import shutil import tarfile from pathlib import Path, PurePosixPath PART_GLOB = 'templates.part*' +C13_BASELINE23_OVERLAY = { + 'README.md': '''# C13 — SubagentStart context semantics + +- Source: `DOCUMENTED` +- Release-gating: `yes` +- Current result: `BLOCKED` +- Qualification package state: `READY_FOR_LIVE_RUN` +- Prepared: `2026-09-02` +- Baseline: `2.3` + +## Objective + +Verify real `SubagentStart` context injection and the documented non-blocking meaning of `continue=false` without conflating those semantics with independent ephemeral parent-thread registration or project-scoped synthetic-agent discovery limitations. + +## Baseline 2.3 transport + +The live harness attempts the aligned project-scoped `fixture_agent` through `codex exec --ephemeral` first. Only the recognized `collab spawn failed: no thread with id` failure may activate a controlled non-ephemeral retry. That retry uses a separate disposable repository containing the project-scoped hook/config but no project-scoped synthetic agent; the child is materialized as `CODEX_HOME/agents/fixture_agent.toml` inside a disposable `CODEX_HOME`. + +`REPRODUCED` still requires one real project-scoped `SubagentStart`, `additionalContext`, `continue=false`, a child echo of an outer-generated proof absent from the root prompt, unchanged repository state, verified session cleanup, and unchanged authentication metadata. + +## Live metadata to record + +Before changing this result to `REPRODUCED`, record the exact Codex version, model slug, OS, permission mode, project trust, fixture commit, transport used, setup/cleanup, sanitized observations, evaluation, and hashes. Do not commit transcripts, credentials, private paths, proof values, session IDs, or unrelated repository data. +''', + 'fixture/README.md': '''# C13 fixture + +The deterministic harness owns this synthetic fixture. + +Ephemeral attempt: +- project-scoped agent file `.codex/agents/fixture_agent.toml`; +- declared agent name `fixture_agent`; +- project-scoped `SubagentStart` hook matcher `^fixture_agent$`. + +Recognized-error fallback only: +- separate disposable Git repository with the same project-scoped hook/config; +- no project-scoped `.codex/agents` child definition; +- synthetic child materialized only as `CODEX_HOME/agents/fixture_agent.toml` inside a disposable `CODEX_HOME`; +- sandbox remains read-only and repository state must remain unchanged. +''', + 'fixture/agent-role.txt': '''Synthetic agent role: fixture_agent. +The ephemeral attempt is project-scoped. The recognized-error fallback materializes the same role only in disposable CODEX_HOME/agents while keeping SubagentStart hooks project-scoped. +''', + 'config/README.md': '''# C13 sandbox configuration — baseline 2.3 + +Use the deterministic live qualification harness rather than an interactive manual session. + +Common requirements: + +```toml +[agents] +enabled = true +max_concurrent_threads_per_session = 2 +``` + +- model: `gpt-5.6-sol`; +- approval: `never`; +- sandbox: `read-only`; +- model-tool network: disabled; +- trusted disposable Git repository; +- real project-scoped `SubagentStart` hook; +- aligned agent filename/name/matcher: `fixture_agent.toml` / `fixture_agent` / `^fixture_agent$`. + +Transport is ephemeral-first. A non-ephemeral retry is allowed only for the recognized parent-thread registration failure. The retry uses a disposable `CODEX_HOME`, home-scoped synthetic agent, temporary file-backed auth symlink, isolated SQLite/log paths, `history.persistence="none"`, mandatory cleanup, and auth-metadata verification. +''', + 'prompt.txt': '''Capability qualification C13: real SubagentStart context semantics. + +Start exactly one configured `fixture_agent` through the real Codex subagent mechanism. Do not simulate the child and do not invoke hook scripts manually. The root session must not use command/file mutation tools. + +The real project-scoped SubagentStart hook injects an opaque proof that is not present in this prompt and returns `continue=false`. Wait for the real child and preserve only the minimal structural result needed to establish whether it received and echoed that injected proof. + +Do not expose credentials, proof values, usernames, home directories, session/thread IDs, private repository URLs, or full transcripts. +''', + 'run-command.txt': '''# Preferred controlled execution from main: +# Actions -> PlanAnvil Codex qualification -> mode=full +# +# Equivalent controller invocation inside the trusted disposable qualification workspace: +python3 tools/live_codex_qualification_harness_v6.py \\ + --root \\ + --source-commit \\ + --run-id \\ + --output \\ + --allow-c13-non-ephemeral-fallback +# +# The permission flag does not force non-ephemeral execution. C13 always runs +# ephemeral first and activates the fallback only for the recognized parent-thread failure. +''', +} + def _safe_member(name: str) -> PurePosixPath: path = PurePosixPath(name) @@ -17,6 +107,32 @@ def _safe_member(name: str) -> PurePosixPath: return path +def _rehash_capability(directory: Path) -> None: + files = {} + for path in sorted(directory.rglob('*')): + if path.is_file() and path.name != 'hashes.json' and '__pycache__' not in path.parts: + files[path.relative_to(directory).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + payload = {'schema_version': '1.0', 'algorithm': 'sha256', 'files': files} + (directory / 'hashes.json').write_text( + json.dumps(payload, indent=2, sort_keys=True) + '\n', encoding='utf-8' + ) + + +def _apply_c13_baseline23_overlay(target_root: Path) -> list[str]: + directory = target_root / 'capabilities' / 'C13' + if not directory.is_dir(): + raise FileNotFoundError('materialized C13 package is missing') + written = [] + for rel, text in C13_BASELINE23_OVERLAY.items(): + target = directory / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding='utf-8') + written.append((Path('capabilities') / 'C13' / rel).as_posix()) + _rehash_capability(directory) + written.append('capabilities/C13/hashes.json') + return written + + def materialize(source_root: Path, target_root: Path, *, force: bool = False) -> list[str]: source_root = source_root.resolve() target_root = target_root.resolve() @@ -41,6 +157,13 @@ def materialize(source_root: Path, target_root: Path, *, force: bool = False) -> raise FileExistsError(f'refusing to overwrite existing capability evidence: {target}') target.write_bytes(data) written.append(rel.as_posix()) + + # Baseline 2.3 intentionally overlays only C13. The stable archive remains the + # historical prepared package source for every other capability, while this + # deterministic overlay keeps C13 transport documentation in lockstep with + # the current live harness and recomputes package hashes before validation. + written.extend(_apply_c13_baseline23_overlay(target_root)) + # The index and package guide are tracked outside the archive and are needed # when materializing into a disposable validation/sandbox root. for rel in (Path('capabilities/index.json'), Path('capabilities/README.md')):