Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/plananvil-codex-qualification.yml
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ jobs:
set -euo pipefail
cd "${QUALIFICATION_REPO}"
set +e
python3 tools/live_codex_qualification_harness.py \
python3 tools/live_codex_qualification_harness_v2.py \
--root "${QUALIFICATION_REPO}" \
--source-commit "${GITHUB_SHA}" \
--run-id "${GITHUB_RUN_ID}" \
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ All notable changes to PlanAnvil are documented here.
- require the full Linux Codex qualification job to pass a system-`bubblewrap` user-namespace probe before C01-C16, so incompatible Podman runners fail fast instead of timing out capability-by-capability;
- update the qualification evidence uploader to the Node-24-native `actions/upload-artifact` v6 immutable SHA;
- use deterministic live-qualification harness setup for C01, C05, C09, and C14 so repository-skill discovery, stale handoff rejection, genuine auto-compaction, and writable auxiliary Git isolation are actually exercised;
- extend deterministic live qualification for C02, C09, C11, C13, C14, and C16 with explicit-only skill policy, current AGENTS precedence evidence, project-scoped SubagentStart semantics, bytecode-free PlanAnvil bootstrap, and real Git signing/hook failure diagnostics; C12 remains isolated for independent runtime verification;
- require production releases to use a GitHub-verified signed annotated tag whose target is reachable from `main`;
- fail the production release gate closed when the release worktree is dirty or Git cleanliness cannot be verified;
- document the controlled self-hosted Codex qualification path and keep the previous sandbox procedure as a manual fallback.
Expand Down
58 changes: 58 additions & 0 deletions tests/test_live_codex_qualification_harness_v2.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
from __future__ import annotations

from pathlib import Path
import unittest


ROOT = Path(__file__).resolve().parents[1]
MODULE_PATH = ROOT / "tools" / "live_codex_qualification_harness_v2.py"


class LiveCodexHarnessV2Tests(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.source = MODULE_PATH.read_text(encoding="utf-8")

def test_wrapper_compiles(self) -> None:
compile(self.source, str(MODULE_PATH), "exec")

def test_exact_target_capabilities(self) -> None:
for capability in ["C02", "C09", "C11", "C13", "C14", "C16"]:
self.assertIn(f'"{capability}"', self.source)
self.assertIn("_ORIGINAL_CAPABILITY_RUNTIME = prior.capability_runtime", self.source)

def test_c02_installs_explicit_only_repository_skill(self) -> None:
self.assertIn("allow_implicit_invocation: false", self.source)
self.assertIn("FIXTURE_SKILL_ACTIVE", self.source)
self.assertIn("implicit_activation_disabled", self.source)
self.assertIn("explicit_activation_available", self.source)

def test_c09_and_c14_disable_python_bytecode(self) -> None:
self.assertIn('os.environ["PYTHONDONTWRITEBYTECODE"] = "1"', self.source)
self.assertIn("return prior._c09_runtime(**common)", self.source)
self.assertIn("return prior._c14_runtime(**common)", self.source)

def test_c11_records_current_docs_and_outer_mapping(self) -> None:
self.assertIn("https://learn.chatgpt.com/docs/agent-configuration/agents-md", self.source)
self.assertIn("AGENTS.override.md", self.source)
self.assertIn("root_to_nested_order_matches", self.source)
self.assertIn("nested_agents_ignored_when_override_exists", self.source)

def test_c13_uses_project_scoped_real_subagent_hook(self) -> None:
self.assertIn("SubagentStart", self.source)
self.assertIn("FIXTURE_SUBAGENT_CONTEXT", self.source)
self.assertIn('"continue": False', self.source)
self.assertIn("CHILD_STARTED_WITH_CONTEXT", self.source)
self.assertNotIn("--dangerously-bypass-approvals-and-sandbox", self.source)

def test_c16_exercises_real_success_signing_and_hook_diagnostics(self) -> None:
self.assertIn("GIT_READY", self.source)
self.assertIn("GIT_SIGNING_BLOCKED", self.source)
self.assertIn("GIT_HOOK_BLOCKED", self.source)
self.assertIn("pre-commit hook failed: C16 fixture hook rejection", self.source)
self.assertIn("gpg: signing failed: C16 fixture signing failure", self.source)
self.assertIn("cleanup_errors is empty", self.source)


if __name__ == "__main__":
unittest.main()
Loading