PlanAnvil Codex qualification #14
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PlanAnvil Codex qualification | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| description: Qualification mode | |
| required: true | |
| default: smoke | |
| type: choice | |
| options: | |
| - smoke | |
| - c13 | |
| - diagnostics | |
| - full | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: plananvil-codex-qualification | |
| cancel-in-progress: false | |
| jobs: | |
| smoke: | |
| name: codex smoke test | |
| if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.mode == 'smoke' | |
| environment: plananvil-codex | |
| runs-on: | |
| - self-hosted | |
| - linux | |
| - x64 | |
| - plananvil | |
| - codex | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Validate controlled runner | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| test "${GITHUB_REF}" = "refs/heads/main" | |
| command -v plananvil-qualification-workspace | |
| command -v codex | |
| codex --version | |
| - name: Run Codex smoke test | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| workspace="$(plananvil-qualification-workspace)" | |
| test -n "${workspace}" | |
| test -d "${workspace}" | |
| cd "${workspace}" | |
| git init -q | |
| git rev-parse --is-inside-work-tree | |
| before="$(git status --porcelain=v1 --untracked-files=all)" | |
| output="$(codex exec --ephemeral 'Do not modify any files. Reply with exactly PLANANVIL_CODEX_SMOKE_OK.')" | |
| printf '%s\n' "${output}" | |
| printf '%s\n' "${output}" | grep -Fq 'PLANANVIL_CODEX_SMOKE_OK' | |
| after="$(git status --porcelain=v1 --untracked-files=all)" | |
| test "${before}" = "${after}" | |
| diagnostics: | |
| name: Codex runner variant matrix | |
| if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.mode == 'diagnostics' | |
| environment: plananvil-codex | |
| runs-on: | |
| - self-hosted | |
| - linux | |
| - x64 | |
| - plananvil | |
| - codex | |
| timeout-minutes: 120 | |
| steps: | |
| - name: Validate controlled runner | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| test "${GITHUB_REF}" = "refs/heads/main" | |
| command -v plananvil-qualification-workspace | |
| command -v codex | |
| command -v git | |
| command -v python3 | |
| codex --version | |
| git --version | |
| python3 --version | |
| - name: Validate Linux Codex sandbox prerequisites | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| command -v bwrap | |
| bwrap --version | |
| bwrap --unshare-user --uid 0 --gid 0 --ro-bind / / /bin/true | |
| - name: Create diagnostic workspace | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| workspace="$(plananvil-qualification-workspace)" | |
| test -n "${workspace}" | |
| test -d "${workspace}" | |
| root="${workspace}/runner-diagnostics-${GITHUB_RUN_ID}" | |
| repo="${root}/source" | |
| fixtures="${root}/fixtures" | |
| artifact="${root}/artifact" | |
| mkdir -p "${repo}" "${fixtures}" "${artifact}" | |
| git -C "${repo}" init -q | |
| git -C "${repo}" remote add origin "https://github.com/${GITHUB_REPOSITORY}.git" | |
| git -C "${repo}" fetch --depth=1 origin "${GITHUB_SHA}" | |
| git -C "${repo}" checkout --detach -q "${GITHUB_SHA}" | |
| test "$(git -C "${repo}" rev-parse HEAD)" = "${GITHUB_SHA}" | |
| echo "DIAGNOSTIC_SOURCE=${repo}" >> "${GITHUB_ENV}" | |
| echo "DIAGNOSTIC_FIXTURES=${fixtures}" >> "${GITHUB_ENV}" | |
| echo "DIAGNOSTIC_ARTIFACT=${artifact}" >> "${GITHUB_ENV}" | |
| - name: Run diagnostic variant matrix | |
| id: diagnose | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| cd "${DIAGNOSTIC_SOURCE}" | |
| set +e | |
| python3 tools/codex_runner_variant_matrix.py \ | |
| --root "${DIAGNOSTIC_FIXTURES}" \ | |
| --output "${DIAGNOSTIC_ARTIFACT}" | |
| rc=$? | |
| set -e | |
| echo "exit_code=${rc}" >> "${GITHUB_OUTPUT}" | |
| exit 0 | |
| - name: Upload sanitized diagnostic matrix | |
| if: always() | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 | |
| with: | |
| name: plananvil-codex-runner-diagnostics-${{ github.run_id }} | |
| path: ${{ env.DIAGNOSTIC_ARTIFACT }} | |
| if-no-files-found: error | |
| retention-days: 14 | |
| - name: Enforce diagnostic harness execution only | |
| if: always() | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| test "${{ steps.diagnose.outputs.exit_code }}" = "0" | |
| test -f "${DIAGNOSTIC_ARTIFACT}/matrix-summary.json" | |
| # Variant observations are intentionally non-gating. This step checks | |
| # only that the diagnostic harness itself completed and produced evidence. | |
| full: | |
| name: live capability qualification | |
| if: >- | |
| github.event_name == 'workflow_dispatch' && | |
| github.ref == 'refs/heads/main' && | |
| (inputs.mode == 'full' || inputs.mode == 'c13') | |
| environment: plananvil-codex | |
| runs-on: | |
| - self-hosted | |
| - linux | |
| - x64 | |
| - plananvil | |
| - codex | |
| timeout-minutes: 180 | |
| steps: | |
| - name: Validate controlled runner | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| test "${GITHUB_REF}" = "refs/heads/main" | |
| command -v plananvil-qualification-workspace | |
| command -v codex | |
| command -v git | |
| command -v python3 | |
| codex --version | |
| git --version | |
| python3 --version | |
| - name: Validate Linux Codex sandbox prerequisites | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if ! command -v bwrap >/dev/null 2>&1; then | |
| echo "::error::System bubblewrap is required on the trusted Linux qualification runner. Install the Debian bubblewrap package instead of relying on the bundled Codex helper." | |
| exit 2 | |
| fi | |
| bwrap --version | |
| if ! bwrap --unshare-user --uid 0 --gid 0 --ro-bind / / /bin/true; then | |
| echo "::error::bubblewrap cannot create the user namespace required by the Codex command sandbox. Fix the Podman/host user-namespace policy before live qualification." | |
| exit 2 | |
| fi | |
| - name: Create trusted main qualification workspace | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| workspace="$(plananvil-qualification-workspace)" | |
| test -n "${workspace}" | |
| test -d "${workspace}" | |
| repo="${workspace}/repo" | |
| artifact="${workspace}/evidence-artifact" | |
| mkdir -p "${repo}" "${artifact}" | |
| git -C "${repo}" init -q | |
| git -C "${repo}" remote add origin "https://github.com/${GITHUB_REPOSITORY}.git" | |
| git -C "${repo}" fetch --depth=1 origin "${GITHUB_SHA}" | |
| git -C "${repo}" checkout --detach -q "${GITHUB_SHA}" | |
| test "$(git -C "${repo}" rev-parse HEAD)" = "${GITHUB_SHA}" | |
| echo "QUALIFICATION_REPO=${repo}" >> "${GITHUB_ENV}" | |
| echo "QUALIFICATION_ARTIFACT=${artifact}" >> "${GITHUB_ENV}" | |
| - name: Materialize and validate evidence templates | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| cd "${QUALIFICATION_REPO}" | |
| python3 tools/prepare_capabilities.py --force | |
| python3 tools/validate_capabilities.py | |
| - name: Run selected live qualification | |
| id: qualify | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| cd "${QUALIFICATION_REPO}" | |
| # Baseline 2.3 permits the narrowly gated C13 fallback in both full and | |
| # C13-only modes. The harness still tries ephemeral first and ignores | |
| # this permission unless the recognized parent-thread failure occurs. | |
| qualification_args=(--allow-c13-non-ephemeral-fallback) | |
| if [ "${{ inputs.mode }}" = "c13" ]; then | |
| qualification_args+=(--only C13) | |
| fi | |
| set +e | |
| python3 tools/live_codex_qualification_harness_v6.py \ | |
| --root "${QUALIFICATION_REPO}" \ | |
| --source-commit "${GITHUB_SHA}" \ | |
| --run-id "${GITHUB_RUN_ID}" \ | |
| --output "${QUALIFICATION_ARTIFACT}" \ | |
| "${qualification_args[@]}" | |
| rc=$? | |
| set -e | |
| echo "exit_code=${rc}" >> "${GITHUB_OUTPUT}" | |
| exit 0 | |
| - name: Upload sanitized capability evidence | |
| if: always() | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 | |
| with: | |
| name: plananvil-codex-evidence-${{ github.run_id }} | |
| path: ${{ env.QUALIFICATION_ARTIFACT }} | |
| if-no-files-found: error | |
| retention-days: 14 | |
| - name: Enforce selected live gate | |
| if: always() | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| test "${{ steps.qualify.outputs.exit_code }}" = "0" |