Add precision Codex runner probes

.github/workflows/plananvil-codex-qualification.yml

@@ -12,6 +12,7 @@ on:
           - smoke
           - c13
           - diagnostics
+          - precision
           - full
 
 permissions:
@@ -151,6 +152,96 @@ jobs:
           # Variant observations are intentionally non-gating. This step checks
           # only that the diagnostic harness itself completed and produced evidence.
 
+  precision:
+    name: Codex runner precision matrix
+    if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.mode == 'precision'
+    environment: plananvil-codex
+    runs-on:
+      - self-hosted
+      - linux
+      - x64
+      - plananvil
+      - codex
+    timeout-minutes: 120
+    steps:
+      - name: Validate controlled runner
+        shell: bash
+        run: |
+          set -euo pipefail
+          test "${GITHUB_REF}" = "refs/heads/main"
+          command -v plananvil-qualification-workspace
+          command -v codex
+          command -v git
+          command -v python3
+          codex --version
+          git --version
+          python3 --version
+
+      - name: Validate Linux Codex sandbox prerequisites
+        shell: bash
+        run: |
+          set -euo pipefail
+          command -v bwrap
+          bwrap --version
+          bwrap --unshare-user --uid 0 --gid 0 --ro-bind / / /bin/true
+
+      - name: Create precision workspace
+        shell: bash
+        run: |
+          set -euo pipefail
+          workspace="$(plananvil-qualification-workspace)"
+          test -n "${workspace}"
+          test -d "${workspace}"
+          root="${workspace}/runner-precision-${GITHUB_RUN_ID}"
+          repo="${root}/source"
+          fixtures="${root}/fixtures"
+          artifact="${root}/artifact"
+          mkdir -p "${repo}" "${fixtures}" "${artifact}"
+
+          git -C "${repo}" init -q
+          git -C "${repo}" remote add origin "https://github.com/${GITHUB_REPOSITORY}.git"
+          git -C "${repo}" fetch --depth=1 origin "${GITHUB_SHA}"
+          git -C "${repo}" checkout --detach -q "${GITHUB_SHA}"
+          test "$(git -C "${repo}" rev-parse HEAD)" = "${GITHUB_SHA}"
+
+          echo "PRECISION_SOURCE=${repo}" >> "${GITHUB_ENV}"
+          echo "PRECISION_FIXTURES=${fixtures}" >> "${GITHUB_ENV}"
+          echo "PRECISION_ARTIFACT=${artifact}" >> "${GITHUB_ENV}"
+
+      - name: Run precision diagnostic matrix
+        id: precision
+        shell: bash
+        run: |
+          set -euo pipefail
+          cd "${PRECISION_SOURCE}"
+          set +e
+          python3 tools/codex_runner_precision_matrix.py \
+            --root "${PRECISION_FIXTURES}" \
+            --output "${PRECISION_ARTIFACT}"
+          rc=$?
+          set -e
+          echo "exit_code=${rc}" >> "${GITHUB_OUTPUT}"
+          exit 0
+
+      - name: Upload sanitized precision matrix
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: plananvil-codex-runner-precision-${{ github.run_id }}
+          path: ${{ env.PRECISION_ARTIFACT }}
+          if-no-files-found: error
+          retention-days: 14
+
+      - name: Enforce precision harness execution only
+        if: always()
+        shell: bash
+        run: |
+          set -euo pipefail
+          test "${{ steps.precision.outputs.exit_code }}" = "0"
+          test -f "${PRECISION_ARTIFACT}/matrix-summary.json"
+          # Variant observations are intentionally non-gating. This step checks
+          # only that the precision harness itself completed and produced evidence.
+
   full:
     name: live capability qualification
     if: >-

tests/test_codex_runner_precision_matrix.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "tools"))
+
+import codex_runner_precision_matrix as precision
+
+WORKFLOW = ROOT / ".github" / "workflows" / "plananvil-codex-qualification.yml"
+SOURCE = ROOT / "tools" / "codex_runner_precision_matrix.py"
+
+
+class CodexRunnerPrecisionMatrixTests(unittest.TestCase):
+    def test_exact_precision_variants(self) -> None:
+        self.assertEqual(len(precision.VARIANT_NAMES), 11)
+        self.assertEqual(precision.VARIANT_NAMES[0], "pretool_json_bash_allow_absolute")
+        self.assertIn("compact_body_after_prefix_two_step_absolute", precision.VARIANT_NAMES)
+        self.assertIn("subagent_non_ephemeral_project_explicit", precision.VARIANT_NAMES)
+        self.assertIn("subagent_non_ephemeral_home_explicit", precision.VARIANT_NAMES)
+
+    def test_pretool_probe_has_absolute_recorder_and_real_deny(self) -> None:
+        source = precision._pretool_script()
+        self.assertNotIn("git rev-parse", source)
+        self.assertIn("Path(sys.argv[1])", source)
+        self.assertIn("cwd_matches_repo", source)
+        self.assertIn("permissionDecisionReason", source)
+        self.assertIn("PLANANVIL_DIAG_PRETOOL_DENY", source)
+
+    def test_compaction_probe_has_absolute_recorder(self) -> None:
+        source = precision._compact_script()
+        self.assertNotIn("git rev-parse", source)
+        self.assertIn("Path(sys.argv[2])", source)
+        text = SOURCE.read_text(encoding="utf-8")
+        self.assertIn("features.token_budget=false", text)
+        self.assertIn("you MUST make a second separate shell tool call", text)
+        self.assertIn("compact_body_after_prefix_single_absolute", text)
+        self.assertIn("compact_total_two_step_absolute", text)
+
+    def test_subagent_opaque_value_is_not_present_in_agent_config(self) -> None:
+        token = "opaque-test-value"
+        self.assertNotIn(token, precision._agent_toml())
+        self.assertIn(token, precision._subagent_script(token))
+        self.assertNotIn("git rev-parse", precision._subagent_script(token))
+        self.assertIn("PLANANVIL_DIAG_CONTEXT_TOKEN=", precision._agent_toml())
+
+    def test_command_observation_uses_aggregated_output_not_command_string(self) -> None:
+        marker = "PLANANVIL_DIAG_HOOK_COMMAND_OK"
+        denied = json.dumps(
+            {
+                "type": "item.completed",
+                "item": {
+                    "type": "command_execution",
+                    "command": f"printf {marker}",
+                    "aggregated_output": "blocked",
+                    "status": "failed",
+                    "exit_code": 1,
+                },
+            }
+        )
+        observed = precision._command_observation(denied, marker)
+        self.assertFalse(observed["marker_output_observed"])
+        self.assertEqual(observed["failed_count"], 1)
+
+    def test_secret_redaction_is_recursive(self) -> None:
+        token = "secret-context"
+        value = {"a": [f"prefix-{token}", {"b": token}]}
+        serialized = json.dumps(precision._redact(value, token), sort_keys=True)
+        self.assertNotIn(token, serialized)
+        self.assertIn("<context-token>", serialized)
+
+    def test_one_variant_failure_does_not_abort_artifact_generation(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            output = Path(tmp)
+            result = precision._run_case(
+                "synthetic",
+                lambda: (_ for _ in ()).throw(TypeError("boom")),
+                output,
+            )
+            self.assertEqual(result["diagnostic_status"], "HARNESS_ERROR")
+            self.assertTrue((output / "synthetic.json").is_file())
+
+    def test_run_matrix_dispatches_all_variants_without_codex(self) -> None:
+        def fake(name: str) -> dict[str, object]:
+            return {"variant": name, "returncode": 0, "diagnostic_status": "TEST"}
+
+        def hook(_root: Path, _output: Path, name: str, *_args: object) -> dict[str, object]:
+            return fake(name)
+
+        def compact(_root: Path, _output: Path, name: str, *_args: object) -> dict[str, object]:
+            return fake(name)
+
+        def subagent(_root: Path, _output: Path, name: str, *_args: object) -> dict[str, object]:
+            return fake(name)
+
+        with tempfile.TemporaryDirectory() as tmp:
+            root, output = Path(tmp) / "runtime", Path(tmp) / "artifact"
+            with (
+                mock.patch.object(precision, "_hook_variant", side_effect=hook) as h,
+                mock.patch.object(precision, "_compact_variant", side_effect=compact) as c,
+                mock.patch.object(precision, "_subagent_variant", side_effect=subagent) as s,
+            ):
+                results = precision.run_matrix(root, output)
+            self.assertEqual([item["variant"] for item in results], list(precision.VARIANT_NAMES))
+            self.assertEqual((h.call_count, c.call_count, s.call_count), (5, 3, 3))
+            self.assertEqual(len(list(output.glob("*.json"))), 11)
+
+    def test_precision_mode_uses_existing_runner_allowed_workflow(self) -> None:
+        workflow = WORKFLOW.read_text(encoding="utf-8")
+        self.assertIn("- precision", workflow)
+        self.assertIn("inputs.mode == 'precision'", workflow)
+        precision_job = workflow[workflow.index("  precision:"):workflow.index("  full:")]
+        self.assertIn("codex_runner_precision_matrix.py", precision_job)
+        for label in ("self-hosted", "linux", "x64", "plananvil", "codex"):
+            self.assertIn(f"- {label}", precision_job)
+        self.assertIn("Variant observations are intentionally non-gating", precision_job)
+        self.assertNotIn("live_codex_qualification_harness_v6.py", precision_job)
+
+    def test_precision_artifact_never_persists_raw_transcripts_or_hook_scripts(self) -> None:
+        source = SOURCE.read_text(encoding="utf-8")
+        self.assertNotIn("write_text(stdout", source)
+        self.assertNotIn("write_text(stderr", source)
+        self.assertIn('root / "scripts"', source)
+        self.assertIn("not uploaded", source)
+        self.assertIn("never a release gate", source)
+
+
+if __name__ == "__main__":
+    unittest.main()