diff --git a/.agents/skills/plan-anvil/references/codex-0.152-contract.md b/.agents/skills/plan-anvil/references/codex-0.152-contract.md index 7eaf500..5fec8c5 100644 --- a/.agents/skills/plan-anvil/references/codex-0.152-contract.md +++ b/.agents/skills/plan-anvil/references/codex-0.152-contract.md @@ -30,3 +30,9 @@ For Codex 0.152, `SubagentStart` may inject `additionalContext`, but `continue: ## Fail-closed rule When the active Codex runtime cannot provide a lifecycle behavior required by the PlanAnvil contract, preserve deterministic evidence and stop with a runtime prerequisite blocker. Do not weaken path, source-immutability, approval, or recovery guarantees to make the run pass. + +## Recovery delivery — source-verified Codex CLI 0.153.4 + +The [official hooks contract](https://developers.openai.com/codex/hooks/) distinguishes stateless compaction events from context-delivery events. `PostCompact` may emit universal control/advisory fields, but its output is not model-visible additional context. PlanAnvil must supply recovery pointers through `SessionStart` matching `source=compact` after compaction; the existing startup/resume/clear matchers remain supported. + +This is a product rule, not a test-only substitution. The PostCompact recovery command reports readiness without embedding the next-action target. The SessionStart command supplies the pointer and asks the model to read canonical files and reconcile Git before continuing. Hooks never substitute for those checks. The pinned `rust-v0.153.4` sources are `codex-rs/hooks/src/events/compact.rs` and `codex-rs/core/src/hook_runtime.rs`. diff --git a/.agents/skills/plan-anvil/tests/test_hooks_and_edge_states.py b/.agents/skills/plan-anvil/tests/test_hooks_and_edge_states.py index 2569770..8a9b798 100644 --- a/.agents/skills/plan-anvil/tests/test_hooks_and_edge_states.py +++ b/.agents/skills/plan-anvil/tests/test_hooks_and_edge_states.py @@ -198,6 +198,35 @@ def test_compaction_accepts_schema_and_git_valid_checkpoint(self) -> None: self.assertIn("Validated checkpoint", message) self.assertIn(str(checkpoint_path.resolve()), message) + def test_compact_recovery_uses_session_start_not_postcompact_context(self) -> None: + with tempfile.TemporaryDirectory() as directory: + repo = self._active_repo(Path(directory), mode="PLAN_EXECUTION") + self._write_valid_checkpoint(repo) + target = "evidence/instruction-map.json" + before = (repo / ".pursue/runs/run/state.json").read_bytes() + advisory = self._hook("plan-anvil-recovery.py", repo, + {"cwd": str(repo), "hook_event_name": "PostCompact", "trigger": "auto"}) + self.assertEqual(set(advisory), {"continue", "systemMessage"}) + self.assertTrue(advisory["continue"]) + self.assertNotIn(target, json.dumps(advisory)) + self.assertIn("SessionStart(source=compact)", advisory["systemMessage"]) + recovery = self._hook("plan-anvil-recovery.py", repo, + {"cwd": str(repo), "hook_event_name": "SessionStart", "source": "compact"}) + output = recovery["hookSpecificOutput"] + self.assertEqual(output["hookEventName"], "SessionStart") + self.assertIn(target, output["additionalContext"]) + self.assertIn("Validated checkpoint", output["additionalContext"]) + self.assertEqual((repo / ".pursue/runs/run/state.json").read_bytes(), before) + + def test_postcompact_reports_invalid_checkpoint_without_unsupported_fields(self) -> None: + with tempfile.TemporaryDirectory() as directory: + repo = self._active_repo(Path(directory)) + output = self._hook("plan-anvil-recovery.py", repo, + {"cwd": str(repo), "hook_event_name": "PostCompact", "trigger": "auto"}) + self.assertEqual(set(output), {"continue", "systemMessage"}) + self.assertIn("invalid", output["systemMessage"]) + self.assertNotIn("hookSpecificOutput", output) + def test_detached_head_with_multiple_containing_branches_is_ambiguous(self) -> None: with tempfile.TemporaryDirectory() as directory: repo = init_repo(Path(directory) / "repo") diff --git a/.codex/hooks.json b/.codex/hooks.json index 8c6fdcd..bf3642a 100644 --- a/.codex/hooks.json +++ b/.codex/hooks.json @@ -6,7 +6,7 @@ { "command": "python3 \"$(git rev-parse --show-toplevel)/.codex/hooks/plan-anvil-recovery.py\"", "commandWindows": "py -3 -c \"import pathlib,runpy,subprocess;r=pathlib.Path(subprocess.check_output(['git','rev-parse','--show-toplevel'],text=True).strip());runpy.run_path(str(r/'.codex/hooks/plan-anvil-recovery.py'),run_name='__main__')\"", - "statusMessage": "Restoring PlanAnvil recovery pointer", + "statusMessage": "Checking PlanAnvil post-compaction recovery readiness", "timeout": 30, "type": "command" } diff --git a/.codex/hooks/plan-anvil-recovery.py b/.codex/hooks/plan-anvil-recovery.py index 73942a6..5f1889f 100644 --- a/.codex/hooks/plan-anvil-recovery.py +++ b/.codex/hooks/plan-anvil-recovery.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json + from plan_anvil_checkpoint import validate_checkpoint_for_run from plan_anvil_hooklib import ( active_run_for_event, @@ -9,11 +11,24 @@ ) +def emit_recovery(event_name: str, message: str) -> None: + if event_name == "PostCompact": + # Codex 0.153.4 PostCompact accepts universal output, not additionalContext. + # Keep this advisory free of recovery targets; SessionStart(source=compact) + # is the model-context delivery boundary, before the continuation request. + print(json.dumps({ + "continue": True, + "systemMessage": message, + }, sort_keys=True)) + else: + context(event_name, message) + + def main() -> int: event = read_event() event_name = str(event.get("hook_event_name") or "SessionStart") if event_has_ambiguous_active_runs(event): - context( + emit_recovery( event_name, "Multiple active PlanAnvil runs match this worktree. Set PLANANVIL_RUN_ID to the intended run before recovery or write-capable work.", ) @@ -36,7 +51,14 @@ def main() -> int: f"next action is {next_action.get('type')} targeting {next_action.get('target')}. " f"{checkpoint_text}" ) - context(event_name, message) + if event_name == "PostCompact": + readiness = "valid" if checkpoint.ok else "invalid; repair required before continuing" + message = ( + f"PlanAnvil checkpoint is {readiness}. " + "SessionStart(source=compact) supplies the recovery pointer; " + "canonical files and Git remain authoritative." + ) + emit_recovery(event_name, message) return 0 diff --git a/.github/workflows/plananvil-codex-qualification.yml b/.github/workflows/plananvil-codex-qualification.yml index 3826eb2..884bace 100644 --- a/.github/workflows/plananvil-codex-qualification.yml +++ b/.github/workflows/plananvil-codex-qualification.yml @@ -346,12 +346,23 @@ jobs: echo "exit_code=${rc}" >> "${GITHUB_OUTPUT}" exit 0 + - name: Pack and verify complete sanitized evidence + id: package_evidence + if: always() && env.QUALIFICATION_ARTIFACT != '' + shell: bash + run: | + set -euo pipefail + cd "${QUALIFICATION_REPO}" + python3 tools/qualification_artifact.py \ + --root "${QUALIFICATION_ARTIFACT}" \ + --output "${QUALIFICATION_ARTIFACT}.zip" + - name: Upload sanitized capability evidence if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: name: plananvil-codex-evidence-${{ github.run_id }} - path: ${{ env.QUALIFICATION_ARTIFACT }} + path: ${{ env.QUALIFICATION_ARTIFACT }}.zip if-no-files-found: error retention-days: 14 @@ -361,3 +372,4 @@ jobs: run: | set -euo pipefail test "${{ steps.qualify.outputs.exit_code }}" = "0" + test "${{ steps.package_evidence.outcome }}" = "success" diff --git a/capabilities/index.json b/capabilities/index.json index aa9cf29..2956b29 100644 --- a/capabilities/index.json +++ b/capabilities/index.json @@ -10,7 +10,7 @@ {"evidence_directory":"capabilities/C07","expected_behavior":"The Git guard rejects the configured unsafe-command corpus","id":"C07","required":true,"result":"BLOCKED","source":"CONTRACT_DEFINED"}, {"evidence_directory":"capabilities/C08","expected_behavior":"PreCompact can stop compaction","id":"C08","required":true,"result":"BLOCKED","source":"DOCUMENTED"}, {"evidence_directory":"capabilities/C09","expected_behavior":"Compaction is allowed after checkpoint creation without a permanent loop","id":"C09","required":true,"result":"BLOCKED","source":"CONTRACT_DEFINED"}, - {"evidence_directory":"capabilities/C10","expected_behavior":"PostCompact and SessionStart can provide recovery context","id":"C10","required":true,"result":"BLOCKED","source":"DOCUMENTED"}, + {"evidence_directory":"capabilities/C10","expected_behavior":"SessionStart supplies recovery context at startup/resume and source=compact; PostCompact is advisory","id":"C10","required":true,"result":"BLOCKED","source":"DOCUMENTED"}, {"evidence_directory":"capabilities/C11","expected_behavior":"Project instructions follow documented directory scope and precedence","id":"C11","required":true,"result":"BLOCKED","source":"DOCUMENTED"}, {"evidence_directory":"capabilities/C12","expected_behavior":"project_doc_max_bytes can truncate automatic instruction loading","id":"C12","required":true,"result":"BLOCKED","source":"DOCUMENTED"}, {"evidence_directory":"capabilities/C13","expected_behavior":"SubagentStart can add context but continue false does not stop subagent startup","id":"C13","required":true,"result":"BLOCKED","source":"DOCUMENTED"}, diff --git a/docs/CODEX_CAPABILITY_BASELINE.md b/docs/CODEX_CAPABILITY_BASELINE.md index 2bb4c8f..95468c3 100644 --- a/docs/CODEX_CAPABILITY_BASELINE.md +++ b/docs/CODEX_CAPABILITY_BASELINE.md @@ -63,7 +63,7 @@ Do not commit session transcripts, credentials, private paths, unrelated Git dat | C07 | The Git guard rejects the configured unsafe-command corpus | CONTRACT_DEFINED | BLOCKED | Git postconditions remain mandatory | | C08 | `PreCompact` can stop compaction | DOCUMENTED | BLOCKED | Delay only until a valid checkpoint exists | | C09 | Compaction is allowed after checkpoint creation without a permanent stop loop | CONTRACT_DEFINED | BLOCKED | Checkpoint, allow, recover and reconcile | -| C10 | `PostCompact` and `SessionStart` can provide recovery context | DOCUMENTED | BLOCKED | Inject only a recovery pointer | +| C10 | `SessionStart` supplies recovery context at startup/resume and after compaction (`source=compact`); `PostCompact` remains advisory | DOCUMENTED | BLOCKED | Inject only a recovery pointer through the supported context event | | C11 | Project instructions follow documented directory scope and precedence | DOCUMENTED | BLOCKED | Explicitly map affected instructions | | C12 | `project_doc_max_bytes` can truncate automatic instruction loading | DOCUMENTED | BLOCKED | Read, size and hash complete files explicitly | | C13 | `SubagentStart` can add context but `continue: false` does not stop subagent startup | DOCUMENTED | BLOCKED | Context/audit only; qualify ephemeral-first with a controlled project-scoped non-ephemeral fallback when the recognized ephemeral parent-thread blocker occurs | diff --git a/docs/CODEX_QUALIFICATION_EXECUTION_AUDIT_2026-09-05.md b/docs/CODEX_QUALIFICATION_EXECUTION_AUDIT_2026-09-05.md index fb468b1..101930a 100644 --- a/docs/CODEX_QUALIFICATION_EXECUTION_AUDIT_2026-09-05.md +++ b/docs/CODEX_QUALIFICATION_EXECUTION_AUDIT_2026-09-05.md @@ -27,6 +27,8 @@ Regression: enter the real compatibility context, seed the same declared project ### C10 configuration provenance +**Superseded assumption:** this historical audit correctly identified root-checkout declaration precedence, but incorrectly assumed that PostCompact itself could deliver model context. The subsequent source review in `CODEX_RECOVERY_DELIVERY_AUDIT_2026-09-05.md` establishes SessionStart(source=compact) as the supported recovery channel. The current fixture narrows that matcher instead of removing SessionStart. The following paragraphs describe the earlier patch, not the active delivery contract. + The previous PostCompact probe removed SessionStart only from a linked planning worktree. Codex still loaded the primary checkout's declarations. The same proof could reach the model through SessionStart, invalidating attribution to PostCompact. Repair: independent source repositories, planning worktrees and random proofs for the two trials. Prepare the second root checkout's hook selection before the fixture commit, product bootstrap and checkpoint. Verify root/local declarations agree and preserve both source and planning state during the probe. Keep the actual product recovery and compaction scripts; never synthesize a live hook event in the qualification runtime. diff --git a/docs/CODEX_RECOVERY_DELIVERY_AUDIT_2026-09-05.md b/docs/CODEX_RECOVERY_DELIVERY_AUDIT_2026-09-05.md new file mode 100644 index 0000000..289b079 --- /dev/null +++ b/docs/CODEX_RECOVERY_DELIVERY_AUDIT_2026-09-05.md @@ -0,0 +1,52 @@ +# Recovery delivery and evidence audit — 2026-09-05 + +## Scope and authority + +Baseline input: PlanAnvil `57909a810b3ceb9deeac8550fd510dc4df28a951`; qualification [run #22](https://github.com/KeyffMS/PlanAnvil/actions/runs/33978465210). C13 was reproduced using its allowed project-scoped fallback and its runtime is unchanged. C09 timed out and C10 lacked exact echo in the isolated after-compaction probe. The previous audit's assumption that PostCompact was a model-context channel was incorrect. + +The implementation specification gives current official Codex documentation priority. This is a product/output-contract correction, not a waiver of capability requirements. No offline test here is evidence that the live capability is reproduced. + +## Source-verified recovery channel + +The [official hooks reference](https://developers.openai.com/codex/hooks/) explicitly limits PostCompact output to common fields and ignores plain stdout. SessionStart matches `source`; `source=compact` runs before the immediate next model request, including compaction in the middle of a turn. + +Pinned Codex source at `rust-v0.153.4`: + +| File | Boundary | +|---|---| +| [hooks/src/events/compact.rs](https://github.com/openai/codex/blob/rust-v0.153.4/codex-rs/hooks/src/events/compact.rs) | StatelessHookOutcome and parsing have no additional-context return channel. | +| [core/src/hook_runtime.rs](https://github.com/openai/codex/blob/rust-v0.153.4/codex-rs/core/src/hook_runtime.rs) | run_post_compact_hooks emits events/control only; pending SessionStart uses record_additional_contexts. | +| [hooks/src/events/session_start.rs](https://github.com/openai/codex/blob/rust-v0.153.4/codex-rs/hooks/src/events/session_start.rs) | SessionStart source matching and additionalContext interpretation. | +| [config/src/loader/mod.rs](https://github.com/openai/codex/blob/rust-v0.153.4/codex-rs/config/src/loader/mod.rs) | Linked-worktree hook declarations come from the corresponding primary checkout. | + +Product behavior: PostCompact reports checkpoint readiness through universal `systemMessage`, with `continue=true`, without including the next-action target. SessionStart supplies the existing pointer/context. This does not change canonical state, checkpoint validation, source ownership or the generator/executor separation. + +C10 tests startup and after-compaction recovery in independent real product fixtures with different opaque targets. The second fixture narrows SessionStart to `^compact$` in the primary checkout BEFORE commit/start/checkpoint. It must observe PreCompact, PostCompact and subsequently SessionStart(source=compact), with no ordinary startup record. PostCompact output must contain no additional context. A matching product-emitted target, exact model echo, allowed tool usage, checkpoint validity and source/planning immutability are all required. The existing two acceptance assertion strings are unchanged. + +## C09/C10 observation and timeout boundary + +Only the C09/C10 process invocations opt into the new observed runner. It retains event counts, the last 128 structural events, exact allowlisted command labels, category-only error counts, and process exit/cleanup status. It never retains raw stdout/stderr, command output, unknown command text, prompts or thread IDs. Lines are size-bounded; malformed/oversized diagnostics are counted. + +On timeout, terminate the owned process group on POSIX or process tree on Windows, wait for it and drain bounded reader threads before proceeding. A cleanup/reader failure blocks the probe. This prevents a CLI launcher timeout from silently leaving its ordinary descendants running during the next capability. It does not claim to control a process which deliberately escapes its owned group/tree. + +C09 retains a bounded structural hook sequence as well. The 900-second timeout, compaction threshold, two-cycle/continuation/checkpoint requirements and rejection of incomplete results remain. We have not proved the cause of the remaining C09 timeout and do not claim to have fixed its runtime completion. C08's intentional negative-path handling and C13's runner are unchanged. + +C10 compares the real product's emitted target against an outer-generated digest, then checks the structured model payload both before and after sanitization. Persisted value-flow diagnostics contain booleans/counts/lengths and fixed classification labels, not proof values or their hashes. Wrong-format echo is diagnosed but not accepted. + +## Complete, limited evidence upload + +The workflow builds a deterministic inner ZIP from validated staged evidence. Packaging requires exactly C01-C16 manifests plus top-level summary/index/guide. Each listed file, including the three hidden fixture files, is hash-checked; unlisted files and symlinks are rejected. ZIP verification independently checks per-capability manifests and the archive manifest, with path and size limits. Removing a hidden member and rewriting only the outer manifest still fails. + +Only the verified archive is uploaded, not the runner workspace or an unrestricted set of dotfiles. Packaging failure also fails the selected gate. The downloaded Actions artifact contains this inner evidence ZIP. No secret-bearing runtime files are added to the allowlist. + +## Regression strategy and handoff + +Executable tests cover real product hook output at PostCompact and SessionStart(source=compact), the installer/Git/start/checkpoint path, root-source matcher isolation, exact echo acceptance, proof-safe diagnostic comparison, actual process timeout/descendant cleanup, partial event retention, malformed/oversized streams and strict archive verification. Existing product and qualification suites remain mandatory. The offline lifecycle driver only treats documented SessionStart output as model context; it does not pretend that arbitrary PostCompact JSON is injectable. + +The existing hosted OS/Python matrix runs `test_qualification_*.py` in addition to core product tests. Qualification results remain BLOCKED until new live evidence is produced. Use the existing workflow on main with `mode=recovery`; that selects C09/C10/C13 through the same runtimes as full and cannot claim a C01-C16 release pass. Do not rerun full until the targeted results are understood. + +## Completion review — 2026-09-06 + +The interrupted upload was recovered and reviewed in PR #32. Unexpected diagnostic reader failures now set a fail-closed flag rather than disappearing in a daemon thread; deeply nested JSON is counted as malformed input without losing later events. Executable tests verify both reader and cleanup failures reject otherwise positive output. The temporary source-snapshot workflow is removed before merge. No live Codex workflow was started during this completion review. + +The Actions artifact now contains an inner verified evidence ZIP. Extract that ZIP before reading qualification-summary.json and the C01-C16 evidence directories. The required hidden fixture files are inside the verified ZIP; unrelated runner dotfiles are never uploaded. diff --git a/docs/IMPLEMENTATION_SPEC.md b/docs/IMPLEMENTATION_SPEC.md index c44403d..79c24d0 100644 --- a/docs/IMPLEMENTATION_SPEC.md +++ b/docs/IMPLEMENTATION_SPEC.md @@ -526,12 +526,15 @@ After compaction: ```text POST_COMPACT_EVENT +→ SESSION_START(source=compact) → RECOVERY_POINTER → READ_MANIFEST_STATE_LOCAL_STATE_CHECKPOINT_PROFILES → RECONCILE_WITH_GIT → CONTINUE_OR_STOP ``` +For Codex CLI 0.153.4, `PostCompact` is a stateless notification/control event; it MUST NOT be relied upon to inject `additionalContext`. The product's `SessionStart` handler matching `source=compact` supplies the recovery pointer to the immediate continuation. Startup/resume recovery uses the same handler with the corresponding source. PostCompact may emit a universal advisory `systemMessage`, without embedding canonical state or the next-action target. + Conversation and hook context are never the source of truth. ## 19. Status model diff --git a/docs/OPENAI_COMPLIANCE.md b/docs/OPENAI_COMPLIANCE.md index 03ff12a..22f9396 100644 --- a/docs/OPENAI_COMPLIANCE.md +++ b/docs/OPENAI_COMPLIANCE.md @@ -82,6 +82,22 @@ PlanAnvil performs a safe reversible probe for refs, branches, linked worktrees, A commit check cannot be skipped while returning `GIT_READY`. Signing and hook failures have explicit blocker results. +### 2.6 Recovery delivery correction — verified 2026-09-05 + +This scoped review targets Codex CLI `0.153.4`; it does not re-date the entire older compliance review above or claim new live capability evidence. + +The [official hooks documentation](https://developers.openai.com/codex/hooks/) specifies that `PostCompact` accepts common output fields, not model-visible `additionalContext`. `SessionStart` matches `source`, including `compact`, and supplies additional developer context before the next model request after compaction, including automatic compaction inside a turn. + +Pinned source confirmation: + +- [hooks/events/compact.rs](https://github.com/openai/codex/blob/rust-v0.153.4/codex-rs/hooks/src/events/compact.rs): `StatelessHookOutcome` and the compact output parser expose control/events, not additional contexts. +- [core/hook_runtime.rs](https://github.com/openai/codex/blob/rust-v0.153.4/codex-rs/core/src/hook_runtime.rs): `run_post_compact_hooks` emits events without recording additional context; `run_pending_session_start_hooks` records the supported context. +- [config/loader/mod.rs](https://github.com/openai/codex/blob/rust-v0.153.4/codex-rs/config/src/loader/mod.rs): linked-worktree hook declarations are resolved from the corresponding primary checkout. + +PlanAnvil now emits a universal readiness advisory for PostCompact and supplies the actual pointer/next-action context from SessionStart. The after-compaction qualification narrows SessionStart to `^compact$` in an independent fixture, excluding ordinary startup as a source of the opaque proof. It retains exact echo, checkpoint, source-immutability and tool restrictions. The two C10 acceptance assertion strings remain unchanged; the earlier description claiming two context-injection events was incorrect and is superseded. + +See `CODEX_RECOVERY_DELIVERY_AUDIT_2026-09-05.md` for the implementation and regression coverage. Offline tests are not live evidence. + ## 3. Current architecture decisions ### 3.1 Generator and executor separation diff --git a/tests/test_live_codex_qualification_c10.py b/tests/test_live_codex_qualification_c10.py index 3dafa9c..5e3f8eb 100644 --- a/tests/test_live_codex_qualification_c10.py +++ b/tests/test_live_codex_qualification_c10.py @@ -39,9 +39,9 @@ def test_postcompact_is_independent_of_session_start_context(self) -> None: hooks = {"hooks": {name: [{"hooks": [{"type": "command", "command": "fixture"}]}] for name in ("SessionStart", "PreCompact", "PostCompact")}} c10.base.json_dump(repo / ".codex/hooks.json", hooks) - self.assertTrue(c10._disable_session_start_for_postcompact(repo)) + self.assertTrue(c10._isolate_compact_session_start(repo)) remaining = c10.base.load_json(repo / ".codex/hooks.json")["hooks"] - self.assertNotIn("SessionStart", remaining) + self.assertEqual(remaining["SessionStart"][0]["matcher"], "^compact$") self.assertEqual(remaining["PreCompact"], hooks["hooks"]["PreCompact"]) self.assertEqual(remaining["PostCompact"], hooks["hooks"]["PostCompact"]) @@ -49,7 +49,7 @@ def test_missing_compaction_handler_fails_isolation(self) -> None: with tempfile.TemporaryDirectory() as tmp: repo = Path(tmp) c10.base.json_dump(repo / ".codex/hooks.json", {"hooks": {"SessionStart": []}}) - self.assertFalse(c10._disable_session_start_for_postcompact(repo)) + self.assertFalse(c10._isolate_compact_session_start(repo)) def test_compaction_trigger_is_qualification_only(self) -> None: source = C10_SOURCE.read_text(encoding="utf-8") diff --git a/tests/test_prepare_capabilities_overlay.py b/tests/test_prepare_capabilities_overlay.py index 25eae04..fd9f3c3 100644 --- a/tests/test_prepare_capabilities_overlay.py +++ b/tests/test_prepare_capabilities_overlay.py @@ -90,6 +90,21 @@ def test_recovery_overlays_do_not_change_expected_assertions(self) -> None: self.assertIn("timeout", (target / "capabilities/C09/fixture/README.md").read_text(encoding="utf-8")) self.assertEqual(validate_capabilities.validate_all(target), []) + def test_c10_documents_supported_context_channel_without_weakening_assertions(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) + prepare_capabilities.materialize(ROOT, target, force=True) + readme = (target / "capabilities/C10/README.md").read_text(encoding="utf-8") + self.assertIn("SessionStart(source=compact)", readme) + self.assertIn("not PostCompact.additionalContext", readme) + fixture = (target / "capabilities/C10/fixture/README.md").read_text(encoding="utf-8") + self.assertIn("^compact$", fixture) + expected = json.loads((target / "capabilities/C10/expected.json").read_text(encoding="utf-8")) + self.assertEqual(expected["assertions"], [ + "Recovery hook injects a pointer/context, not hidden mutable state.", + "Session continuation can reconstruct from canonical files and Git.", + ]) + def test_overlays_do_not_remove_other_capabilities(self) -> None: with tempfile.TemporaryDirectory() as tmp: target = Path(tmp) / "materialized" diff --git a/tests/test_qualification_artifact.py b/tests/test_qualification_artifact.py new file mode 100644 index 0000000..5a54f56 --- /dev/null +++ b/tests/test_qualification_artifact.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +from contextlib import contextmanager +import hashlib +import json +import os +from pathlib import Path +import sys +import tempfile +import unittest +import zipfile + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) +import prepare_capabilities +import qualification_artifact as pack + + +class EvidenceArchiveTests(unittest.TestCase): + @contextmanager + def fixture(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "evidence" + prepare_capabilities.materialize(ROOT, root, force=True) + (root / "qualification-summary.json").write_text(json.dumps({"scope": ["C09", "C10", "C13"], "release_gate_passed": False})) + yield root, Path(tmp) / "package.zip" + + def test_deterministic_archive_contains_all_manifest_listed_hidden_files(self): + with self.fixture() as (root, output): + result = pack.build_archive(root, output) + first = output.read_bytes() + self.assertTrue(result["complete"]) + self.assertEqual(result["manifest_listed_hidden_files"], 3) + with zipfile.ZipFile(output) as z: + for cid in pack.CAPABILITIES: + hashes = json.loads(z.read(f"capabilities/{cid}/hashes.json"))["files"] + for path, digest in hashes.items(): + self.assertEqual(hashlib.sha256(z.read(f"capabilities/{cid}/{path}")).hexdigest(), digest) + pack.build_archive(root, output) + self.assertEqual(output.read_bytes(), first) + + def test_missing_hidden_fixture_is_rejected_before_upload(self): + with self.fixture() as (root, output): + names = [p for p in pack.collect_files(root) if "/.agents/" in p] + self.assertTrue(names) + (root / names[0]).unlink() + with self.assertRaises(pack.ArchiveError): + pack.build_archive(root, output) + self.assertFalse(output.exists()) + + def test_unlisted_secret_cannot_enter_archive(self): + with self.fixture() as (root, output): + (root / ".env").write_text("private=DO_NOT_UPLOAD") + with self.assertRaises(pack.ArchiveError): + pack.build_archive(root, output) + self.assertFalse(output.exists()) + + def test_archive_recheck_detects_missing_member_even_if_outer_manifest_is_rewritten(self): + with self.fixture() as (root, output): + pack.build_archive(root, output) + with zipfile.ZipFile(output) as z: + files = {name: z.read(name) for name in z.namelist()} + hidden = next(name for name in files if "/.agents/" in name) + files.pop(hidden) + manifest = json.loads(files[pack.MANIFEST]) + del manifest["files"][hidden] + files[pack.MANIFEST] = json.dumps(manifest).encode() + with zipfile.ZipFile(output, "w") as z: + for name, data in files.items(): + z.writestr(name, data) + with self.assertRaises(pack.ArchiveError): + pack.verify_archive(output) + + def test_changed_data_cannot_replace_previous_good_archive(self): + with self.fixture() as (root, output): + pack.build_archive(root, output) + previous = output.read_bytes() + (root / "capabilities/C10/actual.sanitized.json").write_text("{}") + with self.assertRaises(pack.ArchiveError): + pack.build_archive(root, output) + self.assertEqual(output.read_bytes(), previous) + + def test_unsafe_archive_member_is_rejected(self): + with self.fixture() as (root, output): + pack.build_archive(root, output) + with zipfile.ZipFile(output, "a") as z: + z.writestr("../not-evidence", "bad") + with self.assertRaises(pack.ArchiveError): + pack.verify_archive(output) + + def test_symlink_is_rejected_instead_of_reading_external_file(self): + if os.name == "nt": + self.skipTest("Windows symlink creation requires privileges not needed by the product") + with self.fixture() as (root, output): + external = root.parent / "external" + external.write_text("private") + (root / "unlisted-link").symlink_to(external) + with self.assertRaises(pack.ArchiveError): + pack.build_archive(root, output) + + def test_workflow_uploads_only_the_verified_archive(self): + source = (ROOT / ".github/workflows/plananvil-codex-qualification.yml").read_text() + self.assertIn('python3 tools/qualification_artifact.py', source) + self.assertIn('path: ${{ env.QUALIFICATION_ARTIFACT }}.zip', source) + self.assertNotIn('include-hidden-files: true', source) + self.assertIn('test "${{ steps.package_evidence.outcome }}" = "success"', source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_qualification_execution_boundaries.py b/tests/test_qualification_execution_boundaries.py index 0a02178..ba94760 100644 --- a/tests/test_qualification_execution_boundaries.py +++ b/tests/test_qualification_execution_boundaries.py @@ -18,6 +18,7 @@ import live_codex_qualification_c10 as c10 import live_codex_qualification_harness_v4 as v4 import live_codex_qualification_harness_v7 as v7 +import qualification_process as process_observation base = v4.base @@ -174,7 +175,7 @@ def driver(args, *, cwd, **kwargs): parsed = json.loads(completed.stdout) if completed.stdout.strip() else {} self.assertIsNot(parsed.get("continue"), False, parsed) context = parsed.get("hookSpecificOutput", {}).get("additionalContext") - if context: + if context and event_name == "SessionStart": contexts.append(context) matches = re.findall(r"evidence/c10-recovery-([0-9a-f]{32})\.json", "\n".join(contexts)) self.assertTrue(matches, "The real product hook did not return the fixture recovery pointer") @@ -197,7 +198,16 @@ def driver(args, *, cwd, **kwargs): events.append({"type": "turn.completed"}) return subprocess.CompletedProcess(args, 0, "\n".join(map(json.dumps, events)), "") - with mock.patch.object(base, "run", side_effect=driver), mock.patch.object( + def observed_driver(args, *, cwd, timeout): + completed = driver(args, cwd=cwd, timeout=timeout) + collector = process_observation.StructuralEvents() + for line in completed.stdout.splitlines(): + collector.accept(line.encode("utf-8")) + events = collector.summary() + events.update({"process_cleanup_ok": True, "timeout": False}) + return process_observation.ProcessResult(completed.returncode, False, events) + + with mock.patch.object(process_observation, "run_observed", side_effect=observed_driver), mock.patch.object( v4, "_write_result", side_effect=lambda **kw: (kw["result"], True) ) as writer: result, _required = c10.run_c10( @@ -210,7 +220,12 @@ def driver(args, *, cwd, **kwargs): self.assertEqual(len(roots), 2) self.assertNotEqual(roots[0], roots[1], "A second invocation must not reuse the first source checkout") self.assertIn("SessionStart", configured_events[0]) - self.assertNotIn("SessionStart", configured_events[1]) + self.assertIn("SessionStart", configured_events[1]) + compact_groups = base.load_json(roots[1] / ".codex/hooks.json")["hooks"]["SessionStart"] + self.assertTrue(all(g["matcher"] == "^compact$" for g in compact_groups)) + flow = evidence["trials"][1]["value_flow"] + self.assertTrue(flow["hook_emitted_expected_target"]) + self.assertTrue(all(h["source"] == "compact" for h in flow["hook_observations"])) self.assertIn("PreCompact", configured_events[1]) self.assertIn("PostCompact", configured_events[1]) self.assertNotEqual(proofs[0], proofs[1], "Recovery probes must have independent proof values") diff --git a/tests/test_qualification_observation.py b/tests/test_qualification_observation.py new file mode 100644 index 0000000..1d346e5 --- /dev/null +++ b/tests/test_qualification_observation.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import time +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) +import qualification_process as process +import qualification_c10_observation as c10obs +import live_codex_qualification_c10 as c10 +import live_codex_qualification_harness_v4 as v4 + + +class StructuralObservationTests(unittest.TestCase): + def test_content_is_not_retained_and_commands_are_exactly_classified(self): + secret = "do-not-store-opaque-private-content" + collector = process.StructuralEvents() + events = [ + {"type": "thread.started", "thread_id": secret}, + {"type": "item.completed", "item": {"type": "command_execution", "command": + "/bin/bash -lc 'cat qualification-payload/segment-01.txt'", "aggregated_output": secret, + "exit_code": 0, "status": "completed"}}, + {"type": "item.completed", "item": {"type": "error", "message": "hook failed " + secret}}, + {"type": secret, "item": {"type": secret, "status": secret}}, + {"type": "turn.failed", "error": {"message": "401 Unauthorized " + secret}}, + ] + for event in events: + collector.accept(json.dumps(event).encode()) + collector.accept(secret.encode(), stderr=True) + result = collector.summary() + self.assertEqual(result["completed_command_items"], 1) + self.assertEqual(result["command_counts"], {"segment_01": 1}) + self.assertEqual(result["error_categories"]["hook_error"], 1) + self.assertEqual(result["error_categories"]["unauthorized"], 1) + self.assertNotIn(secret, json.dumps(result)) + self.assertEqual(process.command_label("cat qualification-payload/segment-01.txt; rm x"), "other") + + def test_event_history_is_bounded_and_totals_remain_complete(self): + collector = process.StructuralEvents() + for _ in range(process.MAX_EVENTS + 50): + collector.accept(b'{"type":"turn.started"}') + result = collector.summary() + self.assertEqual(len(result["event_tail"]), process.MAX_EVENTS) + self.assertTrue(result["event_tail_truncated"]) + self.assertEqual(result["event_types"]["turn.started"], process.MAX_EVENTS + 50) + + def test_malformed_oversized_and_non_object_json_are_bounded(self): + collector = process.StructuralEvents() + stream = io.BytesIO(b'x' * (process.MAX_LINE_BYTES + 30) + b'\ninvalid\n[]\n{"type":"turn.completed"}\n') + collector.read(stream) + result = collector.summary() + self.assertEqual(result["oversized_lines"], 1) + self.assertEqual(result["invalid_json_lines"], 2) + self.assertEqual(result["event_types"], {"turn.completed": 1}) + + def test_successful_real_process_keeps_terminal_event_without_content(self): + secret = "ghp_0123456789abcdefghijklmnop" + program = 'import json,sys; print(json.dumps({"type":"turn.completed","secret":' + repr(secret) + '})); print(' + repr(secret) + ',file=sys.stderr)' + with tempfile.TemporaryDirectory() as tmp: + result = process.run_observed([sys.executable, "-c", program], cwd=Path(tmp), timeout=5) + self.assertEqual(result.returncode, 0) + self.assertFalse(result.timed_out) + self.assertTrue(result.events["process_cleanup_ok"]) + self.assertEqual(result.events["event_types"]["turn.completed"], 1) + self.assertNotIn(secret, json.dumps(result.events)) + + def test_real_timeout_retains_progress_and_kills_owned_descendant(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + marker = root / "escaped-child.txt" + child = f'import time,pathlib; time.sleep(5); pathlib.Path({str(marker)!r}).write_text("escaped")' + parent = ('import subprocess,sys,time,json; ' + 'print(json.dumps({"type":"item.completed","item":{"type":"command_execution",' + '"command":"cat qualification-payload/segment-02.txt","exit_code":0}}),flush=True); ' + f'subprocess.Popen([sys.executable,"-c",{child!r}]); time.sleep(30)') + result = process.run_observed([sys.executable, "-c", parent], cwd=root, timeout=3) + self.assertTrue(result.timed_out) + self.assertTrue(result.events["owned_process_tree_terminated"]) + self.assertTrue(result.events["process_cleanup_ok"]) + self.assertEqual(result.events["command_counts"], {"segment_02": 1}) + time.sleep(5.2) + self.assertFalse(marker.exists(), "A descendant survived timeout and changed state") + + +class RecoveryValueFlowTests(unittest.TestCase): + def test_format_diagnostics_do_not_weaken_exact_echo(self): + proof = "1234" * 8 + for value, classification, passed in [ + ("C10_RECOVERY_ECHO=" + proof, "exact", True), + ("`C10_RECOVERY_ECHO=" + proof + "`", "expected_value_wrong_format", False), + ("C10_RECOVERY_ECHO=" + "5678" * 8, "different_value", False), + ("CONTEXT_MISSING", "no_echo", False), + ]: + with self.subTest(classification=classification): + payload = {"observations": [value]} + result = c10obs.echo_diagnostics(payload, proof) + self.assertEqual(result["classification"], classification) + self.assertEqual(c10._exact_echo(payload, proof), passed) + self.assertNotIn(proof, json.dumps(result)) + result = c10obs.echo_diagnostics({"observations": [], "assertions": [{"evidence": proof}]}, proof) + self.assertTrue(result["expected_value_elsewhere"]) + self.assertFalse(result["exact_observation"]) + + def run_proxy(self, root, proof, emitted, *, exitcode=0, event_name="SessionStart", stdout=None): + hooks = root / ".codex/hooks" + hooks.mkdir(parents=True, exist_ok=True) + # No model, no Codex. Test the generated subprocess contract itself. + subprocess.run(["git", "init", "-q", str(root)], check=True) + (hooks / "proxy.py").write_text(c10obs.proxy_source(proof), encoding="utf-8") + output = stdout if stdout is not None else json.dumps({"hookSpecificOutput": { + "hookEventName": event_name, "additionalContext": f"target evidence/c10-recovery-{emitted}.json"}}) + (hooks / "product.py").write_text(f'import sys; print({output!r}); raise SystemExit({exitcode})\n', encoding="utf-8") + completed = subprocess.run([sys.executable, str(hooks / "proxy.py"), "SessionStart", "product.py"], + cwd=root, input=json.dumps({"hook_event_name": "SessionStart", "source": "compact", "cwd": str(root)}), text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, timeout=10) + self.assertEqual(completed.returncode, exitcode) + self.assertEqual(completed.stdout, output + "\n") + return completed + + def test_proxy_compares_actual_product_value_without_retaining_it(self): + proof = "abcd" * 8 + for emitted, match in [(proof, True), ("1234" * 8, False)]: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.run_proxy(root, proof, emitted) + text = (root / ".pursue/qualification-hook-events.jsonl").read_text() + record = json.loads(text) + self.assertEqual(record["recovery_target_matches_expected"], match) + self.assertTrue(record["output_event_matches_input"]) + self.assertNotIn(proof, text) + self.assertNotIn(emitted, text) + self.assertNotIn(proof, (root / ".codex/hooks/proxy.py").read_text()) + + def test_proxy_preserves_nonzero_exit_invalid_json_and_wrong_event(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.run_proxy(root, "a" * 32, "b" * 32, exitcode=7, stdout="not-json") + record = json.loads((root / ".pursue/qualification-hook-events.jsonl").read_text()) + self.assertEqual(record["returncode"], 7) + self.assertFalse(record["product_stdout_is_json"]) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.run_proxy(root, "a" * 32, "a" * 32, event_name="PostCompact") + record = json.loads((root / ".pursue/qualification-hook-events.jsonl").read_text()) + self.assertFalse(record["output_event_matches_input"]) + + def test_recorder_failure_does_not_change_product_response(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / ".pursue").write_text("blocks logging") + self.run_proxy(root, "a" * 32, "a" * 32) + + def test_probe_observes_before_sanitizing_but_does_not_return_raw_content(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + output = root / "trial-01.json" + proof = "a" * 32 + payload = {"observations": ["C10_RECOVERY_ECHO=" + proof], "blocker": "/home/private/secret"} + def fake_run(args, **kw): + v4.base.json_dump(output, payload) + return process.ProcessResult(0, False, {"process_cleanup_ok": True}) + with mock.patch.object(process, "run_observed", side_effect=fake_run): + parsed, events, error = v4._run_codex_probe(cwd=root, prompt="test", schemas={"trial": root / "schema"}, + results_dir=root, position=1, sandbox="read-only", observe_process=True, + inspect_payload=lambda value: c10obs.echo_diagnostics(value, proof)) + self.assertIsNone(error) + self.assertTrue(events["raw_payload_checks"]["exact_observation"]) + self.assertNotIn(proof, json.dumps(events)) + self.assertEqual(parsed["blocker"], "") + + def test_probe_timeout_keeps_events_and_never_accepts_stale_output(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + v4.base.json_dump(root / "trial-01.json", {"outcome": "PASS"}) + partial = {"process_cleanup_ok": True, "timeout": True, "completed_command_items": 2} + with mock.patch.object(process, "run_observed", return_value=process.ProcessResult(-9, True, partial)): + payload, events, error = v4._run_codex_probe(cwd=root, prompt="test", schemas={"trial": root / "schema"}, + results_dir=root, position=1, sandbox="read-only", observe_process=True) + self.assertEqual(payload, {}) + self.assertEqual(events["completed_command_items"], 2) + self.assertIn("timed out", error) + self.assertFalse((root / "trial-01.json").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_qualification_stream_failures.py b/tests/test_qualification_stream_failures.py new file mode 100644 index 0000000..c78d1b6 --- /dev/null +++ b/tests/test_qualification_stream_failures.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import io +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) +import qualification_process as process +import live_codex_qualification_harness_v4 as v4 + + +class StreamFailureTests(unittest.TestCase): + def test_deeply_nested_json_is_counted_without_losing_next_event(self) -> None: + collector = process.StructuralEvents() + depth = sys.getrecursionlimit() + 100 + stream = io.BytesIO(b"[" * depth + b"0" + b"]" * depth + b'\n{"type":"turn.completed"}\n') + collector.read(stream) + result = collector.summary() + self.assertEqual(result["invalid_json_lines"], 1) + self.assertEqual(result["event_types"], {"turn.completed": 1}) + self.assertFalse(result["reader_failed"]) + + def test_unexpected_reader_exception_sets_fail_closed_flag(self) -> None: + pipe = mock.Mock() + pipe.readline.side_effect = RuntimeError("synthetic private diagnostic") + collector = process.StructuralEvents() + collector.read(pipe) + self.assertTrue(collector.summary()["reader_failed"]) + self.assertNotIn("synthetic private diagnostic", str(collector.summary())) + pipe.close.assert_called_once_with() + + def test_cleanup_and_reader_failures_never_accept_positive_output(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for diagnostic in ( + {"process_cleanup_ok": False}, + {"process_cleanup_ok": True, "reader_failed": True}, + ): + with self.subTest(diagnostic=diagnostic): + def run(args, **kwargs): + v4.base.json_dump(root / "trial-01.json", {"outcome": "PASS"}) + return process.ProcessResult(0, False, diagnostic) + with mock.patch.object(process, "run_observed", side_effect=run): + payload, events, error = v4._run_codex_probe( + cwd=root, prompt="offline", schemas={"trial": root / "schema.json"}, + results_dir=root, position=1, sandbox="read-only", observe_process=True, + ) + self.assertEqual(payload, {}) + self.assertIsNotNone(error) + self.assertEqual(events, diagnostic) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/live_codex_qualification_c10.py b/tools/live_codex_qualification_c10.py index ba8fce1..3d8b53b 100644 --- a/tools/live_codex_qualification_c10.py +++ b/tools/live_codex_qualification_c10.py @@ -8,6 +8,7 @@ import live_codex_qualification_codex0152 as compat import live_codex_qualification_harness_v4 as v4 +from qualification_c10_observation import echo_diagnostics, proxy_source base = v4.base v1 = v4.v1 @@ -63,27 +64,24 @@ def _set_compaction_trigger(repo: Path) -> None: ) -def _disable_session_start_for_postcompact(repo: Path) -> bool: - """Modify the root-checkout hook source, not a linked-worktree override.""" - - hooks_path = repo / ".codex" / "hooks.json" - hooks = base.load_json(hooks_path) - configured = hooks.get("hooks") if isinstance(hooks, dict) else None - if not isinstance(configured, dict): +def _isolate_compact_session_start(repo: Path) -> bool: + """Exclude startup context without disabling the documented recovery channel.""" + path = repo / ".codex" / "hooks.json" + hooks = base.load_json(path) + configured = hooks.get("hooks", {}) + groups = configured.get("SessionStart", []) + if not groups or not configured.get("PreCompact") or not configured.get("PostCompact"): return False - configured.pop("SessionStart", None) - base.json_dump(hooks_path, hooks) - return ( - "SessionStart" not in configured - and bool(configured.get("PreCompact")) - and bool(configured.get("PostCompact")) - ) + for group in groups: + group["matcher"] = "^compact$" + base.json_dump(path, hooks) + return True def _prepare_postcompact_repo(repo: Path) -> None: # Codex rust-v0.153.4 loads linked-worktree hook declarations from the # corresponding root checkout. This runs BEFORE commit/start/checkpoint. - if not _disable_session_start_for_postcompact(repo): + if not _isolate_compact_session_start(repo): raise base.QualificationError("C10 could not isolate the root-checkout hook source") _set_compaction_trigger(repo) @@ -94,7 +92,8 @@ def _postcompact_hooks_isolated(repo: Path, planning: Path) -> bool: hooks = base.load_json(source).get("hooks", {}) return ( source.read_bytes() == local.read_bytes() - and "SessionStart" not in hooks + and bool(hooks.get("SessionStart")) + and all(group.get("matcher") == "^compact$" for group in hooks["SessionStart"]) and bool(hooks.get("PreCompact")) and bool(hooks.get("PostCompact")) ) @@ -125,12 +124,12 @@ def _session_prompt() -> str: def _postcompact_prompt() -> str: - return f"""Exercise the real C10 automatic PostCompact recovery-context path. + return f"""Exercise C10 recovery through SessionStart(source=compact) after genuine automatic compaction. Run exactly one shell-tool command: `cat qualification-payload/segment-01.txt`. Do not combine it with another command and do not make any other tool call before or after it. Continue normally if automatic compaction occurs. After that command and any compaction lifecycle, use only context -already supplied by lifecycle hooks. If it contains a PlanAnvil next-action target matching +already supplied by the SessionStart hook for source=compact (not by ordinary startup). If it contains a PlanAnvil next-action target matching `{C10_TARGET_PREFIX}{C10_TARGET_SUFFIX}`, copy only the opaque portion into one observation exactly as `{C10_ECHO_PREFIX}` and return capability_id C10, trial `postcompact_recovery_context`, outcome PASS. Otherwise return outcome FAIL. Never read `.pursue`, @@ -219,6 +218,7 @@ def run_c10( create_checkpoint=False, segments=1, segment_bytes=C10_SEGMENT_BYTES, + hook_proxy_source=proxy_source(proof), ) fixture_commit = base.git(repo, "rev-parse", "HEAD") _inject_recovery_probe_state(planning, run_root, proof) @@ -241,6 +241,8 @@ def run_c10( position=1, sandbox="read-only", timeout=600, + observe_process=True, + inspect_payload=lambda value: echo_diagnostics(value, proof), ) after_session = base.git_snapshot(planning) source_session_unchanged = source_before_session == base.git_snapshot(repo) @@ -263,6 +265,7 @@ def run_c10( segments=1, segment_bytes=C10_SEGMENT_BYTES, prepare_repo=_prepare_postcompact_repo, + hook_proxy_source=proxy_source(compact_proof), ) compact_fixture_commit = base.git(compact_repo, "rev-parse", "HEAD") postcompact_isolated = _postcompact_hooks_isolated(compact_repo, compact_planning) @@ -286,6 +289,8 @@ def run_c10( compact_limit=C10_COMPACT_LIMIT, compact_scope=v4.COMPACT_SCOPE, timeout=900, + observe_process=True, + inspect_payload=lambda value: echo_diagnostics(value, compact_proof), ) after_compact = base.git_snapshot(compact_planning) source_compact_unchanged = source_before_compact == base.git_snapshot(compact_repo) @@ -297,7 +302,15 @@ def run_c10( session_start = v4._event_records(session_records, "SessionStart") session_context = [item for item in session_start if item.get("additional_context")] session_echo = _exact_echo(session_payload, proof) - session_no_tools = int(session_events.get("completed_command_items") or 0) == 0 + session_value_verified = any( + item.get("returncode") == 0 and item.get("recovery_target_matches_expected") is True + and item.get("output_event_matches_input") is True for item in session_context + ) + session_no_tools = ( + int(session_events.get("completed_command_items") or 0) == 0 + and not any(session_events.get("item_types", {}).get(kind, 0) + for kind in ("file_change", "mcp_tool_call", "web_search", "collab_tool_call", "other")) + ) session_unchanged = bool(before_session) and before_session == after_session and source_session_unchanged session_checkpoint_ok = ( bool(checkpoint_before.get("ok")) and bool(checkpoint_after_session.get("ok")) @@ -307,6 +320,7 @@ def run_c10( and session_error is None and session_payload.get("outcome") == "PASS" and bool(session_context) + and session_value_verified and session_echo and session_no_tools and session_unchanged @@ -316,25 +330,53 @@ def run_c10( precompact = v4._event_records(compact_records, "PreCompact") postcompact = v4._event_records(compact_records, "PostCompact") compact_session_start = v4._event_records(compact_records, "SessionStart") - post_context = [item for item in postcompact if item.get("additional_context")] + compact_startup = [item for item in compact_session_start if item.get("source") != "compact"] + post_context = [item for item in compact_session_start + if item.get("source") == "compact" and item.get("additional_context")] + compact_context_ordered = any( + item in post_context and any(prior_item.get("event") == "PostCompact" + for prior_item in compact_records[:position]) + for position, item in enumerate(compact_records) + ) + postcompact_output_supported = bool(postcompact) and all( + item.get("returncode") == 0 and not item.get("additional_context") + and item.get("product_stdout_is_json") is True + and item.get("system_message_present") is True + and item.get("continue") is True + for item in postcompact + ) compact_stops = [item for item in precompact if item.get("continue") is False] compact_echo = _exact_echo(compact_payload, compact_proof) - compact_one_command = int(compact_events.get("completed_command_items") or 0) == 1 + compact_value_verified = any( + item.get("returncode") == 0 and item.get("recovery_target_matches_expected") is True + and item.get("output_event_matches_input") is True for item in post_context + ) + compact_one_command = ( + int(compact_events.get("completed_command_items") or 0) == 1 + and compact_events.get("command_counts") == {"segment_01": 1} + and not any(compact_events.get("item_types", {}).get(kind, 0) + for kind in ("file_change", "mcp_tool_call", "web_search", "collab_tool_call", "other")) + ) compact_unchanged = bool(before_compact) and before_compact == after_compact and source_compact_unchanged compact_checkpoint_ok = ( bool(checkpoint_before_compact.get("ok")) and bool(checkpoint_after_compact.get("ok")) ) compact_lifecycle = bool(precompact) and bool(postcompact) + compact_processes_ok = all(item.get("returncode") == 0 for item in precompact + postcompact) compact_ok = ( setup_error is None and compact_error is None and compact_payload.get("outcome") == "PASS" and postcompact_isolated - and not compact_session_start + and not compact_startup + and compact_context_ordered + and postcompact_output_supported and compact_lifecycle + and compact_processes_ok and not compact_stops and bool(post_context) + and compact_value_verified and compact_echo and compact_one_command and compact_unchanged @@ -357,8 +399,8 @@ def run_c10( }, { "name": "session_start_supplies_recovery_context", - "status": "PASS" if session_context else ("BLOCKED" if session_error else "FAIL"), - "evidence": f"session_start={len(session_start)}; context_records={len(session_context)}", + "status": "PASS" if session_value_verified else ("BLOCKED" if session_error else "FAIL"), + "evidence": f"session_start={len(session_start)}; context_records={len(session_context)}; expected_target={session_value_verified}", }, { "name": "model_receives_opaque_session_recovery_target_without_tools", @@ -388,6 +430,13 @@ def run_c10( "checkpoint_before": checkpoint_before, "checkpoint_after": checkpoint_after_session, "model_payload_summary": _payload_summary(session_payload), + "value_flow": { + "hook_emitted_expected_target": session_value_verified, + "hook_observations": session_context, + "raw_model": session_events.get("raw_payload_checks", {}), + "sanitized_model": echo_diagnostics(session_payload, proof), + "runtime_delivery_proven": session_value_verified and session_echo, + }, } compact_trial = { @@ -397,11 +446,11 @@ def run_c10( "outcome": "PASS" if compact_ok else ("BLOCKED" if compact_error or setup_error or not compact_lifecycle else "FAIL"), "assertions": [ { - "name": "postcompact_trial_isolated_from_session_start_context", - "status": "PASS" if postcompact_isolated and not compact_session_start else "FAIL", + "name": "after_compaction_trial_excludes_ordinary_startup_context", + "status": "PASS" if postcompact_isolated and not compact_startup else "FAIL", "evidence": ( - f"session_start_removed={str(postcompact_isolated).lower()}; " - f"session_start_records={len(compact_session_start)}" + f"compact_only_matcher={str(postcompact_isolated).lower()}; " + f"ordinary_startup_records={len(compact_startup)}" ), }, { @@ -413,12 +462,12 @@ def run_c10( ), }, { - "name": "postcompact_supplies_recovery_context", - "status": "PASS" if post_context else ("BLOCKED" if not compact_lifecycle else "FAIL"), - "evidence": f"postcompact_context_records={len(post_context)}", + "name": "session_start_compact_supplies_recovery_context", + "status": "PASS" if compact_value_verified else ("BLOCKED" if not compact_lifecycle else "FAIL"), + "evidence": f"session_start_compact_context_records={len(post_context)}; expected_target={compact_value_verified}", }, { - "name": "model_receives_opaque_postcompact_target", + "name": "model_receives_opaque_after_compaction_target", "status": "PASS" if compact_echo and compact_one_command else "FAIL", "evidence": ( f"opaque_echo={str(compact_echo).lower()}; " @@ -440,10 +489,12 @@ def run_c10( }, ], "observations": [ - f"session_start_records={len(compact_session_start)}", + f"session_start_compact_records={len(compact_session_start)}", + f"ordinary_startup_records={len(compact_startup)}", + f"context_after_postcompact={str(compact_context_ordered).lower()}", f"precompact_count={len(precompact)}", f"postcompact_count={len(postcompact)}", - f"postcompact_context_count={len(post_context)}", + f"compact_session_context_count={len(post_context)}", f"continue_false_count={len(compact_stops)}", f"opaque_echo_observed={str(compact_echo).lower()}", f"command_items={int(compact_events.get('completed_command_items') or 0)}", @@ -456,15 +507,27 @@ def run_c10( "checkpoint_before": checkpoint_before_compact, "checkpoint_after": checkpoint_after_compact, "model_payload_summary": _payload_summary(compact_payload), + "value_flow": { + "hook_emitted_expected_target": compact_value_verified, + "hook_observations": post_context, + "raw_model": compact_events.get("raw_payload_checks", {}), + "sanitized_model": echo_diagnostics(compact_payload, compact_proof), + "runtime_delivery_proven": compact_value_verified and compact_echo, + }, "fixture_commit": compact_fixture_commit, "config_evidence": { "model_auto_compact_token_limit": C10_COMPACT_LIMIT, "model_auto_compact_token_limit_scope": v4.COMPACT_SCOPE, "token_budget_disabled_in_isolated_fixture": True, - "session_start_removed_only_for_postcompact_isolation": True, + "startup_context_excluded_with_compact_only_matcher": True, "hook_source": "disposable_root_checkout", + "model_context_event": "SessionStart", + "model_context_source": "compact", + "postcompact_output_supported": postcompact_output_supported, + "compact_hook_processes_ok": compact_processes_ok, "configured_before_bootstrap": True, "independent_recovery_proof": True, + "project_trust_method": "persisted_user_config", }, } @@ -482,7 +545,12 @@ def run_c10( blocker = compact_error or "Deterministic automatic compaction did not reach both PreCompact and PostCompact." elif not compact_ok: result, met = "FAILED", False - blocker = "Real PostCompact did not independently provide coherent PlanAnvil recovery context to the model." + if not compact_value_verified: + blocker = "SessionStart(source=compact) did not emit the expected recovery target; inspect value_flow." + elif not compact_echo: + blocker = "SessionStart(source=compact) emitted the expected target, but the exact model echo was not observed; delivery is unproven." + else: + blocker = "PostCompact probe did not satisfy isolation, tool-use or state-integrity requirements." else: result, met, blocker = "REPRODUCED", True, None @@ -501,7 +569,7 @@ def run_c10( ], blocker=_redact_proofs(blocker, (proof, compact_proof)), summary=( - "C10 reproduced with independent outer-harness-created PlanAnvil runs and product-validated checkpoints; real SessionStart and isolated real PostCompact each supplied recovery context to the model." + "C10 reproduced with independent outer-harness-created PlanAnvil runs and product-validated checkpoints; real startup and compact-source SessionStart each supplied recovery context to the model; PostCompact remained advisory." if met else "C10 deterministic recovery qualification did not completely reproduce both lifecycle context paths." ), diff --git a/tools/live_codex_qualification_harness_v4.py b/tools/live_codex_qualification_harness_v4.py index 01c551a..303651c 100644 --- a/tools/live_codex_qualification_harness_v4.py +++ b/tools/live_codex_qualification_harness_v4.py @@ -120,6 +120,7 @@ def _instrument_hooks( event_to_script: dict[str, str], compact_limit: int | None = None, compact_scope: str | None = None, + proxy_source: str | None = None, ) -> None: hooks_path = repo / ".codex" / "hooks.json" hooks = json.loads(hooks_path.read_text(encoding="utf-8")) @@ -132,7 +133,8 @@ def _instrument_hooks( f'qualification-hook-proxy-v4.py" {event_name} {script_name}' ) _write(hooks_path, json.dumps(hooks, indent=2, sort_keys=True) + "\n") - _write(repo / ".codex" / "hooks" / "qualification-hook-proxy-v4.py", _hook_proxy_source()) + _write(repo / ".codex" / "hooks" / "qualification-hook-proxy-v4.py", + _hook_proxy_source() if proxy_source is None else proxy_source) if compact_limit is not None: _set_compact_config( repo, @@ -187,6 +189,8 @@ def _run_codex_probe( compact_scope: str | None = None, add_dir: Path | None = None, timeout: int = 600, + observe_process: bool = False, + inspect_payload: Callable[[dict[str, Any]], dict[str, Any]] | None = None, ) -> tuple[dict[str, Any], dict[str, Any], str | None]: output = results_dir / f"trial-{position:02d}.json" output.unlink(missing_ok=True) @@ -207,17 +211,27 @@ def _run_codex_probe( f'model_auto_compact_token_limit_scope="{compact_scope or COMPACT_SCOPE}"', ] args.append(prompt) - try: - completed = base.run(args, cwd=cwd, check=False, timeout=timeout) - except subprocess.TimeoutExpired: - return {}, {"timeout": True}, "Codex invocation timed out" - events = base.event_summary(completed.stdout) - if completed.returncode != 0: - return ( - {}, - events, - f"Codex exited {completed.returncode}: {base.sanitize_text(completed.stderr[-2500:])}", - ) + if observe_process: + from qualification_process import run_observed + completed = run_observed(args, cwd=cwd, timeout=timeout) + events = completed.events + if not events.get("process_cleanup_ok") or events.get("reader_failed"): + return {}, events, "Codex process cleanup or diagnostic reader failed" + if completed.timed_out: + return {}, events, "Codex invocation timed out" + if completed.returncode != 0: + return {}, events, f"Codex exited {completed.returncode}; see structural error_categories" + else: + # Preserve the established C06/C08 invocation contract, including C08's + # intentional negative compaction-stop trial. C13 has its own runner. + try: + completed = base.run(args, cwd=cwd, check=False, timeout=timeout) + except subprocess.TimeoutExpired: + return {}, {"timeout": True}, "Codex invocation timed out" + events = base.event_summary(completed.stdout) + if completed.returncode != 0: + return ({}, events, + f"Codex exited {completed.returncode}: {base.sanitize_text(completed.stderr[-2500:])}") if not output.is_file(): return {}, events, "Codex did not produce the structured output file" try: @@ -226,6 +240,10 @@ def _run_codex_probe( return {}, events, f"Codex produced invalid structured output: {exc}" if not isinstance(payload, dict): return {}, events, "Codex structured output was not a JSON object" + if inspect_payload is not None: + # Compare in memory before redaction; the callback returns only structural + # diagnostics. Raw model content is never included in event evidence. + events["raw_payload_checks"] = inspect_payload(payload) return base.sanitize(payload), events, None @@ -450,6 +468,7 @@ def _start_active_run( segments: int, segment_bytes: int, prepare_repo: Callable[[Path], None] | None = None, + hook_proxy_source: str | None = None, ) -> tuple[Path, str]: v1._install_plananvil_release(root, repo) _instrument_hooks( @@ -462,6 +481,7 @@ def _start_active_run( }, compact_limit=compact_limit, compact_scope=COMPACT_SCOPE, + proxy_source=hook_proxy_source, ) _write(repo / "README.md", "Deterministic PlanAnvil compaction qualification fixture.\n") payload_dir = repo / "qualification-payload" @@ -834,6 +854,7 @@ def _c09_runtime( compact_limit=C09_COMPACT_LIMIT, compact_scope=COMPACT_SCOPE, timeout=900, + observe_process=True, ) after = base.git_snapshot(planning) records = _read_hook_records(planning) @@ -906,6 +927,13 @@ def _c09_runtime( ], "blocker": completion_blocker, "event_summary": events, + "hook_timeline": [ + {key: item[key] for key in ("event", "returncode", "continue", "additional_context") + if key in item} + for item in records[-128:] + if item.get("event") in {"PreToolUse", "SessionStart", "PreCompact", "PostCompact"} + ], + "hook_timeline_truncated": len(records) > 128, "git_before": before, "git_after": after, "checkpoint_before": checkpoint_before, @@ -915,6 +943,8 @@ def _c09_runtime( "model_auto_compact_token_limit": C09_COMPACT_LIMIT, "model_auto_compact_token_limit_scope": COMPACT_SCOPE, "runtime_cli_override": True, + "project_trust_method": "persisted_user_config", + "process_observation": "bounded_structural_jsonl", }, } @@ -996,4 +1026,4 @@ def main(argv: list[str] | None = None) -> int: if __name__ == "__main__": - raise SystemExit(main()) \ No newline at end of file + raise SystemExit(main()) diff --git a/tools/prepare_capabilities.py b/tools/prepare_capabilities.py index fabf184..7a5cea2 100644 --- a/tools/prepare_capabilities.py +++ b/tools/prepare_capabilities.py @@ -137,7 +137,7 @@ Two real compaction cycles, coherent checkpoint/Git state, and subsequent real tool use remain required. They are not sufficient when Codex times out or fails to return a completed positive structured C09 result. Partial event counts cannot turn an incomplete invocation into REPRODUCED. -The deliberately low fixture threshold is not a product default. This correction does not silently retune it or weaken C08's intentional negative stop trial. Record a remaining timeout as BLOCKED. +The deliberately low fixture threshold is not a product default. This correction does not silently retune it or weaken C08's intentional negative stop trial. Record a remaining timeout as BLOCKED. Retain bounded content-free event and hook sequences, command labels/counts, error categories, and process completion/owned-tree cleanup status. Never persist raw stdout/stderr or use partial progress as proof of successful completion. ''', 'run-command.txt': '''# Existing controlled workflow: main -> recovery for C09/C10/C13. # The recovery driver selects the same v7 capability runtime used by full. @@ -147,26 +147,45 @@ } C10_ISOLATION_OVERLAY = { - 'fixture/README.md': '''# C10 independent recovery fixtures + 'README.md': """# C10 — Recovery context through SessionStart -Prepare each fixture deterministically through the actual installer, product start command, checkpoint creator and checkpoint validator. The model must not construct its own prerequisites. +- Source: `DOCUMENTED_AND_SOURCE_VERIFIED` +- Release-gating: `yes` +- Current result: `BLOCKED` +- Qualification package state: `READY_FOR_LIVE_RUN` +- Target runtime: Codex CLI `0.153.4`; record the executed version -SessionStart and PostCompact use independent source repositories, planning worktrees and opaque next-action targets. For PostCompact, remove SessionStart from the root checkout's hook declarations BEFORE the fixture commit and bootstrap. Codex 0.153.4 redirects linked-worktree hook declarations to the root checkout; changing only planning/.codex/hooks.json is not isolation. +Verify file/Git-based recovery at startup and immediately after genuine automatic compaction. The model-visible channel is SessionStart(source=compact), not PostCompact.additionalContext. PostCompact reports readiness using universal output fields. Canonical files/Git remain authoritative. -The live runtime must observe the actual product recovery hook and an exact opaque echo without unauthorized file/tool reads. Keep both source and planning state unchanged and redact proof values from persisted evidence. Offline command/lifecycle-driver tests verify setup, not live capability reproduction. -''', - 'config/README.md': '''# C10 configuration provenance +The two acceptance assertions remain unchanged. Exact opaque echo, actual lifecycle execution, no unauthorized tool reads, valid checkpoints, and source/planning immutability are mandatory. A declared model PASS without exact echo is not evidence of delivery. +""", + 'prompt.txt': """Exercise the installed PlanAnvil recovery handler at ordinary SessionStart and, independently, SessionStart(source=compact) after automatic compaction. PostCompact is a stateless advisory, not a context channel. -Use the same v7 runner live-auth/persisted-trust context as C08/C09. Do not copy or restore authentication tokens. The runner config.toml is restored byte-for-byte after the probe. +Use deterministic outer-harness fixtures and independent opaque targets. Never manually invoke a hook, read the target from files, invent it, or accept a model-declared PASS without an exact externally checked echo. Preserve source/planning state and checkpoint validity. +""", + 'fixture/README.md': """# C10 independent recovery fixtures -The SessionStart fixture retains the product startup hook. The independent PostCompact fixture retains PreCompact and PostCompact, excludes SessionStart at the primary hook source, and checks that the linked checkout has identical declarations. Source configuration is prepared before product snapshots/checkpoints, not mutated afterwards. +Prepare each fixture through the actual installer, product start command, checkpoint creator and checkpoint validator. The model must not construct its own prerequisites. -Sandbox remains read-only, approval remains never, model-tool network access remains disabled. A low auto-compaction threshold and token_budget=false apply only to the disposable compaction fixture, not product defaults. -''', +Startup and after-compaction probes use independent source repositories, planning worktrees and opaque next-action targets. Narrow SessionStart to ^compact$ in the second root checkout BEFORE the fixture commit and bootstrap. Do not remove this supported recovery channel. Codex 0.153.4 redirects linked-worktree hook declarations to the root checkout; changing only planning/.codex/hooks.json is not isolation. + +Require PreCompact -> PostCompact -> SessionStart(source=compact), no ordinary startup record, a matching target actually emitted by the product, and an exact model echo without unauthorized file/tool reads. PostCompact emits a universal readiness advisory and no next-action target. + +Keep source and planning state unchanged. Persist only boolean/hash-comparison results and bounded structural observations, never the opaque values. Offline drivers model only documented SessionStart context delivery; they do not constitute live capability evidence. +""", + 'config/README.md': """# C10 configuration provenance + +Use the v7 runner live-auth/persisted-trust context. Do not copy or restore authentication tokens; restore runner config.toml byte-for-byte after the probe. + +The startup fixture retains the product SessionStart hook. The independent after-compaction fixture retains PreCompact and PostCompact and narrows SessionStart to ^compact$ at the primary hook source. Check that the linked checkout has identical declarations. Prepare this BEFORE product snapshots/checkpoints. Model context comes only from SessionStart(source=compact), whose handler output is checked before examining the model echo. + +Sandbox remains read-only, approval never, and model-tool network disabled. Low auto-compaction threshold and token_budget=false apply only to disposable fixtures, not product defaults. Preserve the process's bounded structural progress on timeout and kill its owned process group/tree before continuing the harness. +""", 'run-command.txt': C09_COMPLETION_OVERLAY['run-command.txt'], } + def _safe_member(name: str) -> PurePosixPath: path = PurePosixPath(name) if path.is_absolute() or '..' in path.parts or not path.parts or path.parts[0] != 'capabilities': @@ -230,6 +249,11 @@ def materialize(source_root: Path, target_root: Path, *, force: bool = False) -> written.extend(_apply_overlay(target_root, 'C06', C06_CODEX0152_OVERLAY)) written.extend(_apply_overlay(target_root, 'C09', C09_COMPLETION_OVERLAY)) written.extend(_apply_overlay(target_root, 'C10', C10_ISOLATION_OVERLAY)) + c10_expected_path = target_root / 'capabilities/C10/expected.json' + c10_expected = json.loads(c10_expected_path.read_text(encoding='utf-8')) + c10_expected['title'] = 'Recovery context via SessionStart at startup and after compaction' + c10_expected_path.write_text(json.dumps(c10_expected, indent=2, sort_keys=True) + '\n', encoding='utf-8') + _rehash_capability(c10_expected_path.parent) written.extend(_apply_overlay(target_root, 'C13', C13_BASELINE23_OVERLAY)) # The index and package guide are tracked outside the archive and are needed diff --git a/tools/qualification_artifact.py b/tools/qualification_artifact.py new file mode 100644 index 0000000..09f6310 --- /dev/null +++ b/tools/qualification_artifact.py @@ -0,0 +1,160 @@ +"""Package exactly the validated evidence, including manifest-listed dotfiles. + +Only this archive is uploaded, never an expanded runner workspace. Verification +checks both the archive manifest and every capability's existing hashes.json. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import stat +import tempfile +import zipfile +from typing import Any + +import validate_capabilities + +CAPABILITIES = tuple(f"C{i:02d}" for i in range(1, 17)) +TOP_FILES = {"qualification-summary.json", "capabilities/index.json", "capabilities/README.md"} +MANIFEST = "archive-manifest.json" +MAX_FILE_BYTES = 16 * 1024 * 1024 +MAX_TOTAL_BYTES = 128 * 1024 * 1024 + + +class ArchiveError(ValueError): + pass + + +def _safe_name(value: str) -> str: + p = PurePosixPath(value) + if (not value or p.is_absolute() or ".." in p.parts or "\\" in value + or ":" in value or p.as_posix() != value or "\x00" in value): + raise ArchiveError("Invalid evidence archive member name") + return value + + +def _json(payload: bytes) -> dict[str, Any]: + value = json.loads(payload) + if not isinstance(value, dict): + raise ArchiveError("Evidence manifest is not an object") + return value + + +def _expected_files(payloads: dict[str, bytes]) -> set[str]: + expected = set(TOP_FILES) + index = _json(payloads["capabilities/index.json"]) + records = index.get("capabilities", []) + if not isinstance(records, list) or any(not isinstance(item, dict) for item in records): + raise ArchiveError("Invalid evidence index records") + ids = [item.get("id") for item in records] + if any(not isinstance(cid, str) for cid in ids) or sorted(ids) != list(CAPABILITIES): + raise ArchiveError("Evidence index must contain C01-C16 exactly once") + for cid in CAPABILITIES: + prefix = f"capabilities/{cid}/" + name = prefix + "hashes.json" + expected.add(name) + hashes = _json(payloads[name]) + if hashes.get("algorithm") != "sha256" or not isinstance(hashes.get("files"), dict): + raise ArchiveError("Invalid capability hash manifest") + for relative, digest in hashes["files"].items(): + member = prefix + _safe_name(relative) + expected.add(member) + if not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest): + raise ArchiveError("Invalid capability digest") + if member not in payloads or hashlib.sha256(payloads[member]).hexdigest() != digest: + raise ArchiveError(f"Missing or changed evidence member: {member}") + return expected + + +def collect_files(root: Path) -> dict[str, bytes]: + root = root.resolve() + payloads: dict[str, bytes] = {} + total = 0 + for path in sorted(root.rglob("*")): + if path.is_symlink(): + raise ArchiveError("Evidence archive must not contain symlinks") + if not path.is_file(): + continue + name = _safe_name(path.relative_to(root).as_posix()) + size = path.stat().st_size + total += size + if size > MAX_FILE_BYTES or total > MAX_TOTAL_BYTES: + raise ArchiveError("Evidence exceeds archive size limits") + payloads[name] = path.read_bytes() + try: + expected = _expected_files(payloads) + except KeyError as exc: + raise ArchiveError("Required evidence metadata is missing") from exc + if set(payloads) != expected: + raise ArchiveError("Evidence file set differs from the explicit capability manifests") + errors = validate_capabilities.validate_all(root) + if errors: + raise ArchiveError("Capability validation failed before packaging: " + "; ".join(errors)) + return payloads + + +def verify_archive(path: Path) -> dict[str, Any]: + with zipfile.ZipFile(path) as archive: + infos = archive.infolist() + names = [_safe_name(info.filename) for info in infos] + if len(names) != len(set(names)): + raise ArchiveError("Duplicate archive member") + if any(info.is_dir() or stat.S_ISLNK(info.external_attr >> 16) for info in infos): + raise ArchiveError("Only regular evidence files are permitted") + if any(info.file_size > MAX_FILE_BYTES for info in infos) or sum(x.file_size for x in infos) > MAX_TOTAL_BYTES: + raise ArchiveError("Evidence exceeds archive size limits") + payloads = {name: archive.read(name) for name in names} + try: + manifest = _json(payloads.pop(MANIFEST)) + expected = _expected_files(payloads) + except KeyError as exc: + raise ArchiveError("Required evidence metadata is missing from archive") from exc + hashes = {name: hashlib.sha256(data).hexdigest() for name, data in sorted(payloads.items())} + if set(payloads) != expected or manifest.get("files") != hashes or manifest.get("algorithm") != "sha256" or manifest.get("schema_version") != "1.0": + raise ArchiveError("Archive manifest or complete evidence file set mismatch") + return {"schema_version": "1.0", "complete": True, "file_count": len(payloads), + "manifest_listed_hidden_files": sum(any(part.startswith(".") for part in PurePosixPath(name).parts) for name in payloads)} + + +def build_archive(root: Path, output: Path) -> dict[str, Any]: + root, output = root.resolve(), output.resolve() + if output == root or output.is_relative_to(root): + raise ArchiveError("Archive output must be outside the staged evidence directory") + payloads = collect_files(root) + manifest = {"schema_version": "1.0", "algorithm": "sha256", "files": { + name: hashlib.sha256(data).hexdigest() for name, data in sorted(payloads.items())}} + payloads[MANIFEST] = (json.dumps(manifest, sort_keys=True, indent=2) + "\n").encode("utf-8") + output.parent.mkdir(parents=True, exist_ok=True) + fd, name = tempfile.mkstemp(prefix=".evidence-", suffix=".zip", dir=output.parent) + os.close(fd) + tmp = Path(name) + try: + with zipfile.ZipFile(tmp, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for relative, content in sorted(payloads.items()): + info = zipfile.ZipInfo(relative, date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.create_system = 3 + info.external_attr = (stat.S_IFREG | 0o644) << 16 + archive.writestr(info, content) + result = verify_archive(tmp) + os.replace(tmp, output) + return result + finally: + tmp.unlink(missing_ok=True) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + print(json.dumps(build_archive(args.root, args.output), sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/qualification_c10_observation.py b/tools/qualification_c10_observation.py new file mode 100644 index 0000000..40a808f --- /dev/null +++ b/tools/qualification_c10_observation.py @@ -0,0 +1,102 @@ +"""C10 value-flow checks. Never retain proof values or model/hook text.""" +from __future__ import annotations + +import hashlib +import re +from typing import Any + +PREFIX = "C10_RECOVERY_ECHO=" + + +def echo_diagnostics(payload: dict[str, Any], proof: str) -> dict[str, Any]: + observations = payload.get("observations") + strings = [x for x in observations if isinstance(x, str)] if isinstance(observations, list) else [] + expected = PREFIX + proof + exact = any(s.strip() == expected for s in strings) + candidates = [m for s in strings for m in re.findall(r"C10_RECOVERY_ECHO=([^\s`\"'<>]*)", s)] + + def contains(value: Any) -> bool: + if isinstance(value, str): + return proof in value + if isinstance(value, list): + return any(contains(x) for x in value) + if isinstance(value, dict): + return any(contains(x) for x in value.values()) + return False + + present = any(proof in s for s in strings) + return { + "observations_is_array": isinstance(observations, list), + "exact_observation": exact, + "expected_value_in_observations": present, + "expected_value_elsewhere": contains({k: v for k, v in payload.items() if k != "observations"}), + "echo_prefix_observed": any(PREFIX in s for s in strings), + "candidate_count": len(candidates), + "candidate_lengths": [len(x) for x in candidates[:8]], + "matching_candidate_count": sum(x == proof for x in candidates), + "classification": ( + "exact" if exact else "expected_value_wrong_format" if present + else "different_value" if candidates else "no_echo" + ), + } + + +def proxy_source(proof: str) -> str: + # A one-way digest is embedded before bootstrap, not the secret itself. + # The proxy forwards the PRODUCT stdout/stderr and exit code unchanged. + digest = hashlib.sha256(proof.encode("utf-8")).hexdigest() + return _PROXY.replace("__EXPECTED_DIGEST__", digest) + + +_PROXY = r'''from __future__ import annotations +import hashlib, json, subprocess, sys +from pathlib import Path +import re + +expected_digest = "__EXPECTED_DIGEST__" +event_name, script_name = sys.argv[1], sys.argv[2] +raw = sys.stdin.read() +root = Path(subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip()) +completed = subprocess.run([sys.executable, str(root / ".codex/hooks" / script_name)], + input=raw, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) +try: + event = json.loads(raw) +except (ValueError, TypeError): + event = {} +try: + parsed = json.loads(completed.stdout) + parsed_valid = isinstance(parsed, dict) +except (ValueError, TypeError): + parsed = {} + parsed_valid = False +record = {"event": event_name, "returncode": completed.returncode, + "product_stdout_is_json": parsed_valid, + "product_stderr_present": bool(completed.stderr)} +if isinstance(event, dict) and isinstance(event.get("source"), str) and event["source"] in {"startup", "resume", "clear", "compact"}: + record["source"] = event["source"] +if isinstance(parsed, dict): + record["system_message_present"] = bool(parsed.get("systemMessage")) + if "continue" in parsed: + record["continue"] = parsed["continue"] is not False + output = parsed.get("hookSpecificOutput") + if isinstance(output, dict): + text = output.get("additionalContext") + text = text if isinstance(text, str) else "" + candidates = re.findall(r"evidence/c10-recovery-([0-9a-f]{32})\.json", text) + record.update({"additional_context": bool(text), "context_chars": len(text), + "recovery_target_count": len(candidates), + "recovery_target_matches_expected": len(candidates) == 1 and + hashlib.sha256(candidates[0].encode("utf-8")).hexdigest() == expected_digest, + "output_event_matches_input": isinstance(event, dict) and + output.get("hookEventName") == event.get("hook_event_name") == event_name}) +try: + log = root / ".pursue/qualification-hook-events.jsonl" + log.parent.mkdir(parents=True, exist_ok=True) + with log.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") +except Exception: + pass +sys.stdout.write(completed.stdout) +sys.stderr.write(completed.stderr) +raise SystemExit(completed.returncode) +''' diff --git a/tools/qualification_process.py b/tools/qualification_process.py new file mode 100644 index 0000000..3a53fff --- /dev/null +++ b/tools/qualification_process.py @@ -0,0 +1,258 @@ +"""Bounded, content-free observations for the C09/C10 Codex processes. + +No transcript, command output, prompt, thread ID or diagnostic message is persisted. +Only allowlisted structural labels, counts, booleans and exit codes leave this module. +""" +from __future__ import annotations + +from collections import Counter, deque +from dataclasses import dataclass +import json +import os +from pathlib import Path +import shlex +import signal +import subprocess +import threading +import time +from typing import Any, BinaryIO + +MAX_LINE_BYTES = 1_048_576 +MAX_EVENTS = 128 +EVENT_TYPES = frozenset({ + "thread.started", "turn.started", "turn.completed", "turn.failed", "error", + "item.started", "item.updated", "item.completed", "hook.started", "hook.completed", +}) +ITEM_TYPES = frozenset({ + "agent_message", "reasoning", "command_execution", "file_change", "mcp_tool_call", + "web_search", "todo_list", "error", "collab_tool_call", "context_compaction", +}) +STATUSES = frozenset({"in_progress", "completed", "failed", "declined", "cancelled"}) +COMMANDS = { + **{f"cat qualification-payload/segment-{i:02d}.txt": f"segment_{i:02d}" for i in range(1, 5)}, + "git status --porcelain=v1 --untracked-files=all": "git_status", + "git rev-parse HEAD": "git_head", +} + + +def command_label(command: Any) -> str: + if not isinstance(command, str): + return "other" + try: + words = shlex.split(command) + if len(words) == 3 and Path(words[0]).name in {"bash", "sh", "zsh"} and words[1] in {"-c", "-lc"}: + words = shlex.split(words[2]) + # No substring matching: extra commands cannot masquerade as a fixture read. + for text, label in COMMANDS.items(): + if words == shlex.split(text): + return label + except ValueError: + pass + return "other" + + +def error_category(value: Any) -> str: + text = str(value).lower() + if "refresh token was already used" in text: + return "auth_refresh_reused" + if "401" in text or "unauthorized" in text: + return "unauthorized" + if "429" in text or "rate limit" in text: + return "rate_limit" + if "hook" in text: + return "hook_error" + if "compact" in text: + return "compaction_error" + if "sandbox" in text or "permission denied" in text: + return "sandbox_error" + return "other" + + +class StructuralEvents: + def __init__(self) -> None: + self.events: Counter[str] = Counter() + self.items: Counter[str] = Counter() + self.commands: Counter[str] = Counter() + self.errors: Counter[str] = Counter() + self.tail: deque[dict[str, Any]] = deque(maxlen=MAX_EVENTS) + self.completed_commands = 0 + self.file_changes = 0 + self.invalid_lines = 0 + self.oversized_lines = 0 + self.stderr_lines = 0 + self.reader_failed = False + self._start = time.monotonic() + self._lock = threading.Lock() + + def accept(self, raw: bytes, *, stderr: bool = False) -> None: + with self._lock: + if stderr: + self.stderr_lines += 1 + if raw.strip(): + self.errors["stderr_" + error_category(raw.decode("utf-8", "replace"))] += 1 + return + try: + event = json.loads(raw) + except (ValueError, UnicodeError, RecursionError): + self.invalid_lines += 1 + return + if not isinstance(event, dict): + self.invalid_lines += 1 + return + kind = event.get("type") + kind = kind if isinstance(kind, str) and kind in EVENT_TYPES else "other" + self.events[kind] += 1 + row: dict[str, Any] = {"event": kind, "elapsed_ms": round((time.monotonic() - self._start) * 1000)} + item = event.get("item") + if isinstance(item, dict): + item_kind = item.get("type") + item_kind = item_kind if isinstance(item_kind, str) and item_kind in ITEM_TYPES else "other" + self.items[item_kind] += 1 + row["item"] = item_kind + status = item.get("status") + if isinstance(status, str) and status in STATUSES: + row["status"] = status + code = item.get("exit_code") + if type(code) is int: + row["exit_code"] = code + if item_kind == "command_execution": + label = command_label(item.get("command")) + row["command"] = label + if kind == "item.completed": + self.completed_commands += 1 + self.commands[label] += 1 + if item_kind == "file_change" and kind == "item.completed": + self.file_changes += 1 + if item_kind == "error": + label = error_category(item.get("message", item.get("text", ""))) + self.errors[label] += 1 + row["error_category"] = label + if kind in {"error", "turn.failed"}: + label = error_category(event.get("error", event.get("message", ""))) + self.errors[label] += 1 + row["error_category"] = label + self.tail.append(row) + + def read(self, pipe: BinaryIO, *, stderr: bool = False) -> None: + try: + # readline(size) bounds even a malformed stream without newlines. + while True: + raw = pipe.readline(MAX_LINE_BYTES + 1) + if not raw: + return + if len(raw) > MAX_LINE_BYTES: + with self._lock: + self.oversized_lines += 1 + while raw and not raw.endswith(b"\n"): + raw = pipe.readline(MAX_LINE_BYTES + 1) + continue + self.accept(raw, stderr=stderr) + except Exception: + # A diagnostic failure must never disappear in a daemon reader. + # Persist only this flag; the caller blocks qualification. + with self._lock: + self.reader_failed = True + finally: + pipe.close() + + def summary(self) -> dict[str, Any]: + with self._lock: + return { + "event_types": dict(sorted(self.events.items())), + "item_types": dict(sorted(self.items.items())), + "completed_command_items": self.completed_commands, + "completed_file_change_items": self.file_changes, + "error_events": self.events["error"], + "command_counts": dict(sorted(self.commands.items())), + "error_categories": dict(sorted(self.errors.items())), + "event_tail": list(self.tail), + "event_tail_truncated": sum(self.events.values()) > MAX_EVENTS, + "invalid_json_lines": self.invalid_lines, + "oversized_lines": self.oversized_lines, + "stderr_lines": self.stderr_lines, + "reader_failed": self.reader_failed, + } + + +@dataclass(frozen=True) +class ProcessResult: + returncode: int + timed_out: bool + events: dict[str, Any] + + +def _kill_owned_tree(process: subprocess.Popen[bytes]) -> bool: + if os.name == "posix": + try: + os.killpg(process.pid, signal.SIGKILL) + return True + except ProcessLookupError: + return True + except OSError: + process.kill() + return False + try: + killed = subprocess.run( + ["taskkill", "/PID", str(process.pid), "/T", "/F"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10, check=False, + ) + if killed.returncode == 0: + return True + except (OSError, subprocess.TimeoutExpired): + pass + if process.poll() is None: + process.kill() + return False + + +def run_observed(args: list[str], *, cwd: Path, timeout: float) -> ProcessResult: + """Kill the owned process group/tree on timeout and retain partial event structure.""" + if timeout <= 0: + raise ValueError("Process timeout must be positive") + options: dict[str, Any] = {"start_new_session": True} if os.name == "posix" else { + "creationflags": subprocess.CREATE_NEW_PROCESS_GROUP, + } + started = time.monotonic() + process = subprocess.Popen(args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **options) + assert process.stdout is not None and process.stderr is not None + collector = StructuralEvents() + readers = [ + threading.Thread(target=collector.read, args=(process.stdout,), daemon=True), + threading.Thread(target=collector.read, args=(process.stderr,), kwargs={"stderr": True}, daemon=True), + ] + for reader in readers: + reader.start() + timed_out = False + cleanup_ok = True + tree_terminated = False + try: + try: + process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + tree_terminated = True + cleanup_ok = _kill_owned_tree(process) + process.wait(timeout=10) + for reader in readers: + reader.join(timeout=1) + if any(reader.is_alive() for reader in readers): + # A launcher may exit while a child still owns stdout/stderr. + tree_terminated = True + cleanup_ok = _kill_owned_tree(process) and cleanup_ok + for reader in readers: + reader.join(timeout=5) + except BaseException: + _kill_owned_tree(process) + process.wait(timeout=10) + for reader in readers: + reader.join(timeout=5) + raise + events = collector.summary() + events.update({ + "timeout": timed_out, + "process_returncode": process.returncode, + "process_elapsed_ms": round((time.monotonic() - started) * 1000), + "owned_process_tree_terminated": tree_terminated, + "process_cleanup_ok": cleanup_ok and not any(reader.is_alive() for reader in readers), + }) + return ProcessResult(process.returncode, timed_out, events)