Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/plananvil-codex-qualification.yml
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ jobs:
set -euo pipefail
cd "${QUALIFICATION_REPO}"
set +e
python3 tools/live_codex_qualification_harness_v3.py \
python3 tools/live_codex_qualification_harness_v4.py \
--root "${QUALIFICATION_REPO}" \
--source-commit "${GITHUB_SHA}" \
--run-id "${GITHUB_RUN_ID}" \
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ All notable changes to PlanAnvil are documented here.
- use deterministic live-qualification harness setup for C01, C05, C09, and C14 so repository-skill discovery, stale handoff rejection, genuine auto-compaction, and writable auxiliary Git isolation are actually exercised;
- extend deterministic live qualification for C02, C09, C11, C13, C14, and C16 with explicit-only skill policy, current AGENTS precedence evidence, project-scoped SubagentStart semantics, bytecode-free PlanAnvil bootstrap, and real Git signing/hook failure diagnostics;
- make C12 a deterministic runtime byte-budget probe with redundant `project_doc_max_bytes` enforcement, secret head/tail markers, zero-tool automatic-loading evidence, and outer PlanAnvil full-file hash verification;
- make C06, C08, and C09 deterministic live probes using the real PlanAnvil PreToolUse/PreCompact/PostCompact hooks, explicit postcondition evidence, low-limit `body_after_prefix` auto-compaction triggers, checkpoint repair, repeated compaction, and post-second-compaction continuation checks;
- require production releases to use a GitHub-verified signed annotated tag whose target is reachable from `main`;
- fail the production release gate closed when the release worktree is dirty or Git cleanliness cannot be verified;
- document the controlled self-hosted Codex qualification path and keep the previous sandbox procedure as a manual fallback.
Expand Down
63 changes: 63 additions & 0 deletions tests/test_live_codex_qualification_harness_v4.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
from __future__ import annotations

from pathlib import Path
import unittest


ROOT = Path(__file__).resolve().parents[1]
MODULE_PATH = ROOT / "tools" / "live_codex_qualification_harness_v4.py"
WORKFLOW_PATH = ROOT / ".github" / "workflows" / "plananvil-codex-qualification.yml"


class LiveCodexHarnessV4Tests(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.source = MODULE_PATH.read_text(encoding="utf-8")
cls.workflow = WORKFLOW_PATH.read_text(encoding="utf-8")

def test_exact_target_capabilities(self) -> None:
self.assertIn('TARGET_CAPABILITIES = {"C06", "C08", "C09"}', self.source)
self.assertIn("_ORIGINAL_CAPABILITY_RUNTIME = prior.capability_runtime", self.source)

def test_c06_uses_real_pretooluse_hook_and_outer_postcondition(self) -> None:
self.assertIn('"PreToolUse": "plan-anvil-guard.py"', self.source)
self.assertIn('item.get("tool_name") == "apply_patch"', self.source)
self.assertIn("outer_non_intercepted_postcondition", self.source)
self.assertIn("git_postcondition_detected", self.source)
self.assertIn("mutation_origin=outer qualification harness outside Codex hook lifecycle", self.source)

def test_c08_uses_real_auto_compact_stop_and_repair(self) -> None:
self.assertIn("C08_COMPACT_LIMIT = 200", self.source)
self.assertIn('COMPACT_SCOPE = "body_after_prefix"', self.source)
self.assertIn('"PreCompact": "plan-anvil-compaction.py"', self.source)
self.assertIn("automatic_compaction_without_valid_checkpoint", self.source)
self.assertIn("automatic_compaction_after_checkpoint_repair", self.source)
self.assertIn("stop_reason_mentions_checkpoint", self.source)
self.assertIn("_create_checkpoint(planning=planning, run_root=run_root)", self.source)

def test_c09_requires_two_real_compactions_and_continuation(self) -> None:
self.assertIn("C09_COMPACT_LIMIT = 1000", self.source)
self.assertIn("len(pre) >= 2 and len(post) >= 2", self.source)
self.assertIn("_continued_after_second_postcompact", self.source)
self.assertIn("tool_use_after_second_postcompact", self.source)
self.assertIn("checkpoint_after_valid", self.source)

def test_compact_limit_is_redundantly_applied_at_runtime(self) -> None:
self.assertIn("model_auto_compact_token_limit={compact_limit}", self.source)
self.assertIn('model_auto_compact_token_limit_scope="{compact_scope or COMPACT_SCOPE}"', self.source)
self.assertIn('model_auto_compact_token_limit_scope = "{scope}"', self.source)

def test_safety_boundary_is_not_weakened(self) -> None:
self.assertNotIn("--dangerously-bypass-approvals-and-sandbox", self.source)
self.assertNotIn("danger-full-access", self.source)
self.assertNotIn("--privileged", self.source)
self.assertNotIn("SYS_ADMIN", self.source)
self.assertIn('sandbox="read-only"', self.source)
self.assertIn('sandbox="workspace-write"', self.source)

def test_full_workflow_uses_v4_wrapper(self) -> None:
self.assertIn("python3 tools/live_codex_qualification_harness_v4.py", self.workflow)


if __name__ == "__main__":
unittest.main()
Loading