Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 91 additions & 0 deletions .github/workflows/plananvil-codex-qualification.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ on:
options:
- smoke
- c13
- diagnostics
- full

permissions:
Expand Down Expand Up @@ -60,6 +61,96 @@ jobs:
after="$(git status --porcelain=v1 --untracked-files=all)"
test "${before}" = "${after}"

diagnostics:
name: Codex runner variant matrix
if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.mode == 'diagnostics'
environment: plananvil-codex
runs-on:
- self-hosted
- linux
- x64
- plananvil
- codex
timeout-minutes: 120
steps:
- name: Validate controlled runner
shell: bash
run: |
set -euo pipefail
test "${GITHUB_REF}" = "refs/heads/main"
command -v plananvil-qualification-workspace
command -v codex
command -v git
command -v python3
codex --version
git --version
python3 --version

- name: Validate Linux Codex sandbox prerequisites
shell: bash
run: |
set -euo pipefail
command -v bwrap
bwrap --version
bwrap --unshare-user --uid 0 --gid 0 --ro-bind / / /bin/true

- name: Create diagnostic workspace
shell: bash
run: |
set -euo pipefail
workspace="$(plananvil-qualification-workspace)"
test -n "${workspace}"
test -d "${workspace}"
root="${workspace}/runner-diagnostics-${GITHUB_RUN_ID}"
repo="${root}/source"
fixtures="${root}/fixtures"
artifact="${root}/artifact"
mkdir -p "${repo}" "${fixtures}" "${artifact}"

git -C "${repo}" init -q
git -C "${repo}" remote add origin "https://github.com/${GITHUB_REPOSITORY}.git"
git -C "${repo}" fetch --depth=1 origin "${GITHUB_SHA}"
git -C "${repo}" checkout --detach -q "${GITHUB_SHA}"
test "$(git -C "${repo}" rev-parse HEAD)" = "${GITHUB_SHA}"

echo "DIAGNOSTIC_SOURCE=${repo}" >> "${GITHUB_ENV}"
echo "DIAGNOSTIC_FIXTURES=${fixtures}" >> "${GITHUB_ENV}"
echo "DIAGNOSTIC_ARTIFACT=${artifact}" >> "${GITHUB_ENV}"

- name: Run diagnostic variant matrix
id: diagnose
shell: bash
run: |
set -euo pipefail
cd "${DIAGNOSTIC_SOURCE}"
set +e
python3 tools/codex_runner_variant_matrix.py \
--root "${DIAGNOSTIC_FIXTURES}" \
--output "${DIAGNOSTIC_ARTIFACT}"
rc=$?
set -e
echo "exit_code=${rc}" >> "${GITHUB_OUTPUT}"
exit 0

- name: Upload sanitized diagnostic matrix
if: always()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: plananvil-codex-runner-diagnostics-${{ github.run_id }}
path: ${{ env.DIAGNOSTIC_ARTIFACT }}
if-no-files-found: error
retention-days: 14

- name: Enforce diagnostic harness execution only
if: always()
shell: bash
run: |
set -euo pipefail
test "${{ steps.diagnose.outputs.exit_code }}" = "0"
test -f "${DIAGNOSTIC_ARTIFACT}/matrix-summary.json"
# Variant observations are intentionally non-gating. This step checks
# only that the diagnostic harness itself completed and produced evidence.

full:
name: live capability qualification
if: >-
Expand Down
102 changes: 0 additions & 102 deletions .github/workflows/plananvil-codex-runner-diagnostics.yml

This file was deleted.

21 changes: 13 additions & 8 deletions tests/test_codex_runner_variant_matrix.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@

import codex_runner_variant_matrix as matrix

WORKFLOW = ROOT / ".github" / "workflows" / "plananvil-codex-runner-diagnostics.yml"
WORKFLOW = ROOT / ".github" / "workflows" / "plananvil-codex-qualification.yml"
STANDALONE_WORKFLOW = ROOT / ".github" / "workflows" / "plananvil-codex-runner-diagnostics.yml"
SOURCE = ROOT / "tools" / "codex_runner_variant_matrix.py"


Expand Down Expand Up @@ -104,17 +105,21 @@ def test_subagent_matrix_varies_runtime_and_agent_type(self) -> None:
self.assertIn("_prepare_isolated_codex_home", source)
self.assertIn("_cleanup_isolated_codex_home", source)

def test_workflow_is_main_only_self_hosted_and_diagnostic_only(self) -> None:
def test_diagnostics_route_through_existing_runner_allowed_workflow(self) -> None:
workflow = WORKFLOW.read_text(encoding="utf-8")
self.assertFalse(STANDALONE_WORKFLOW.exists())
self.assertIn("workflow_dispatch", workflow)
self.assertIn("- diagnostics", workflow)
self.assertIn("inputs.mode == 'diagnostics'", workflow)
self.assertIn("github.ref == 'refs/heads/main'", workflow)
diagnostics = workflow[workflow.index(" diagnostics:"):workflow.index(" full:")]
for label in ("self-hosted", "linux", "x64", "plananvil", "codex"):
self.assertIn(f"- {label}", workflow)
self.assertIn("codex_runner_variant_matrix.py", workflow)
self.assertIn("plananvil-codex-runner-diagnostics-${{ github.run_id }}", workflow)
self.assertIn("Variant observations are intentionally non-gating", workflow)
self.assertNotIn("release_gate_passed", workflow)
self.assertNotIn("live_codex_qualification_harness_v6.py", workflow)
self.assertIn(f"- {label}", diagnostics)
self.assertIn("codex_runner_variant_matrix.py", diagnostics)
self.assertIn("plananvil-codex-runner-diagnostics-${{ github.run_id }}", diagnostics)
self.assertIn("Variant observations are intentionally non-gating", diagnostics)
self.assertNotIn("release_gate_passed", diagnostics)
self.assertNotIn("live_codex_qualification_harness_v6.py", diagnostics)

def test_diagnostic_output_does_not_persist_raw_transcripts(self) -> None:
source = SOURCE.read_text(encoding="utf-8")
Expand Down