Fail-closed security posture, hardware gates in code, auto-merge fixes, appliance preflight, reboot setting

.github/workflows/auto-merge-trusted.yml

@@ -3,38 +3,135 @@ name: Auto-merge trusted PRs
 # dangerous-triggers is ignored on `on:` below: this workflow runs only the GitHub API via
 # github-script on the trusted-author allowlist; never checks out or executes
 # PR code, which is what makes workflow_run dangerous.
+#
+# Two required checks live in two workflows (`ci-gate` in CI, `operability-gate` in
+# operability), so this listens to BOTH. Whichever finishes first sees the other's gate
+# still pending and exits; the second workflow's completion is the retry. After a merge
+# it dispatches CI and operability on master, because a merge made with GITHUB_TOKEN
+# emits no `push` run (so `full-suite` would never run on the merge commit).
 on:  # zizmor: ignore[dangerous-triggers] -- see comment above
   workflow_run:
-    workflows: ["CI"]
+    workflows: ["CI", "operability"]
     types: [completed]
 
 permissions:
   contents: read
 
 jobs:
   merge-trusted-pr:
-    if: >
-      github.event.workflow_run.conclusion == 'success' &&
-      github.event.workflow_run.event == 'pull_request'
+    # No conclusion filter here: decide() enforces success per required check, and
+    # skipping non-success completions would drop the only retry a waiting PR has.
+    if: github.event.workflow_run.event == 'pull_request'
     runs-on: ubuntu-latest
+    # Both workflows can finish together; serialise so the loser sees the PR merged.
+    concurrency:
+      group: auto-merge-${{ github.event.workflow_run.head_sha }}
+      cancel-in-progress: false
     permissions:
       contents: write
       pull-requests: write
+      checks: read
+      statuses: read
+      actions: write
     steps:
       - name: Merge trusted PRs targeting master
         uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b  # v7.1.0
         with:
           script: |
+            // decide(requiredContexts, checkRuns, statuses) -> { decision, pending, failed }
+            // decision is "merge" | "wait" | "fail". Pure: no API calls.
+            //   checkRuns: [{ id, name, status, conclusion }] (all runs on the head SHA)
+            //   statuses:  [{ context, state }] (combined status: latest per context)
+            // A required context counts as passed ONLY on conclusion/state "success".
+            // The newest check run (highest id) per name wins, so a re-run supersedes.
+            // A context with no run and no status yet is "pending" (its workflow has
+            // not reported; that workflow's completion re-triggers this job).
+            //
+            // Worked examples (required = ["ci-gate", "operability-gate"]):
+            //  1. all green: runs [{id:1,name:"ci-gate",status:"completed",conclusion:"success"},
+            //     {id:2,name:"operability-gate",status:"completed",conclusion:"success"}], statuses []
+            //     -> { decision: "merge", pending: [], failed: [] }
+            //  2. one pending (PR #47): runs [{id:1,name:"ci-gate",status:"completed",
+            //     conclusion:"success"}, {id:2,name:"operability-gate",status:"in_progress",
+            //     conclusion:null}] -> { decision: "wait", pending: ["operability-gate"], failed: [] }
+            //  3. one failed: runs [{id:1,name:"ci-gate",status:"completed",conclusion:"failure"},
+            //     {id:2,name:"operability-gate",status:"completed",conclusion:"success"}]
+            //     -> { decision: "fail", pending: [], failed: ["ci-gate (failure)"] }
+            function decide(requiredContexts, checkRuns, statuses) {
+              const pending = [];
+              const failed = [];
+              for (const ctx of requiredContexts) {
+                const runs = checkRuns.filter((r) => r.name === ctx).sort((a, b) => b.id - a.id);
+                const status = statuses.find((s) => s.context === ctx);
+                if (runs.length) {
+                  const run = runs[0];
+                  if (run.status !== "completed") pending.push(ctx);
+                  else if (run.conclusion !== "success") failed.push(`${ctx} (${run.conclusion})`);
+                } else if (status) {
+                  if (status.state === "pending") pending.push(ctx);
+                  else if (status.state !== "success") failed.push(`${ctx} (${status.state})`);
+                } else {
+                  pending.push(ctx);
+                }
+              }
+              const decision = failed.length ? "fail" : pending.length ? "wait" : "merge";
+              return { decision, pending, failed };
+            }
+
+            // reviewGate(author, reviews, headSha) -> { ok, reason }
+            // Pure. The owner merges directly. Any other trusted author (Copilot,
+            // whose PR content is steered by issue text) additionally needs the
+            // owner's latest substantive review to be APPROVED on the exact head
+            // SHA; a later push, CHANGES_REQUESTED or DISMISSED review resets it.
+            const reviewOwner = "Amantux";
+            function reviewGate(author, reviews, headSha) {
+              if (author === reviewOwner) return { ok: true, reason: "owner" };
+              let latest = null;
+              for (const r of reviews) {
+                if (!r.user || r.user.login !== reviewOwner) continue;
+                if (r.state === "COMMENTED" || r.state === "PENDING") continue;
+                latest = r; // listReviews is chronological
+              }
+              if (!latest || latest.state !== "APPROVED") {
+                return { ok: false, reason: `no ${reviewOwner} approval` };
+              }
+              if (latest.commit_id !== headSha) {
+                return { ok: false, reason: `${reviewOwner} approval is on an older commit` };
+              }
+              return { ok: true, reason: "approved" };
+            }
+
             const owner = context.repo.owner;
             const repo = context.repo.repo;
             const allowedAuthors = new Set(["Amantux", "Copilot"]);
+            const fallbackRequired = ["ci-gate", "operability-gate"];
             const prs = context.payload.workflow_run.pull_requests || [];
 
             if (!prs.length) {
-              core.info("No pull requests attached to this CI run.");
+              core.info("No pull requests attached to this workflow run.");
               return;
             }
 
+            async function requiredContextsFor(branch) {
+              try {
+                const { data } = await github.rest.repos.getStatusChecksProtection({
+                  owner,
+                  repo,
+                  branch,
+                });
+                if (data.contexts && data.contexts.length) return data.contexts;
+                return fallbackRequired;
+              } catch (error) {
+                // GITHUB_TOKEN cannot be granted `administration: read`, so this is
+                // usually 403/404; fall back to the known branch-protection list.
+                if (error.status === 403 || error.status === 404) {
+                  core.info(`Branch protection not readable (${error.status}); using ${fallbackRequired.join(", ")}.`);
+                  return fallbackRequired;
+                }
+                throw error;
+              }
+            }
+
             for (const prRef of prs) {
               const number = prRef.number;
               const { data: pr } = await github.rest.pulls.get({
@@ -60,15 +157,77 @@ jobs:
                 continue;
               }
 
+              const required = await requiredContextsFor(pr.base.ref);
+              // Only GitHub Actions check runs count, so another app cannot post a
+              // look-alike gate name.
+              const checkRuns = (
+                await github.paginate(github.rest.checks.listForRef, {
+                  owner,
+                  repo,
+                  ref: pr.head.sha,
+                  per_page: 100,
+                })
+              ).filter((r) => r.app && r.app.slug === "github-actions");
+              const { data: combined } = await github.rest.repos.getCombinedStatusForRef({
+                owner,
+                repo,
+                ref: pr.head.sha,
+                per_page: 100,
+              });
+              const verdict = decide(required, checkRuns, combined.statuses || []);
+              if (verdict.decision === "wait") {
+                core.notice(`Not merging #${number} yet: waiting for ${verdict.pending.join(", ")}; the other workflow's completion will re-run this.`);
+                continue;
+              }
+              if (verdict.decision === "fail") {
+                core.notice(`Not merging #${number}: required check(s) not successful: ${verdict.failed.join(", ")}.`);
+                continue;
+              }
+
+              const reviews = await github.paginate(github.rest.pulls.listReviews, {
+                owner,
+                repo,
+                pull_number: number,
+                per_page: 100,
+              });
+              const review = reviewGate(pr.user.login, reviews, pr.head.sha);
+              if (!review.ok) {
+                core.notice(`Not merging #${number}: ${review.reason} on ${pr.head.sha}.`);
+                continue;
+              }
+
               try {
                 await github.rest.pulls.merge({
                   owner,
                   repo,
                   pull_number: number,
                   merge_method: "squash",
+                  sha: pr.head.sha,
                 });
                 core.info(`Merged trusted PR #${number}.`);
               } catch (error) {
                 core.warning(`Unable to merge #${number}: ${error.message}`);
+                continue;
+              }
+
+              // A GITHUB_TOKEN merge emits no `push` run; workflow_dispatch is the one
+              // event this token can trigger, so run the master-only lanes explicitly.
+              const dispatchFailures = [];
+              for (const workflowId of ["ci.yml", "operability.yml"]) {
+                try {
+                  await github.rest.actions.createWorkflowDispatch({
+                    owner,
+                    repo,
+                    workflow_id: workflowId,
+                    ref: "master",
+                  });
+                  core.info(`Dispatched ${workflowId} on master after merging #${number}.`);
+                } catch (error) {
+                  dispatchFailures.push(`${workflowId}: ${error.message}`);
+                }
+              }
+              if (dispatchFailures.length) {
+                // Fail red: without these runs full-suite never covers the merge.
+                core.setFailed(`Merged #${number} but could not dispatch master lanes: ${dispatchFailures.join("; ")}`);
               }
             }

.github/workflows/ci.yml

@@ -8,7 +8,9 @@ on:
 
 concurrency:
   group: ci-${{ github.ref }}
-  cancel-in-progress: true
+  # PR runs supersede each other; dispatched master runs (post-merge full-suite /
+  # operability) must never cancel one another or a merge ends up with no result.
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read

.github/workflows/hardware-destructive.yml

@@ -83,21 +83,39 @@ jobs:
 
       - run: pip install -e '.[dev]'
 
+      # Real mode refuses to start without an API token (openblade/config.py
+      # raises RealHardwareDisabledError). Mint a throwaway one per run, masked.
+      - name: Mint a per-run API token
+        run: |
+          set -euo pipefail
+          token=$(openssl rand -hex 32)
+          echo "::add-mask::$token"
+          echo "OPENBLADE_API_TOKEN=$token" >> "$GITHUB_ENV"
+
       # A rig lane must never go green by skipping: the rig tests skip when
       # these are unset, so refuse to run without them. OPENBLADE_CHANGER_DEVICE
       # and OPENBLADE_DRIVE_DEVICES come from the runner host's environment.
-      - name: Preflight — rig configuration present
+      - name: Preflight — rig ready (python -m tools.hardware.preflight)
+        # Read-only tool (env, devices, inquiry serials, changer, allowlist,
+        # appliance login). It must never run armed: blank the job-level enable.
+        env:
+          OPENBLADE_REAL_HARDWARE_ENABLED: ""
         run: |
           set -euo pipefail
-          missing=()
-          for v in OPENBLADE_APPLIANCE_URL OPENBLADE_APPLIANCE_USER OPENBLADE_APPLIANCE_PASSWORD OPENBLADE_CHANGER_DEVICE OPENBLADE_DRIVE_DEVICES; do
-            if [ -z "${!v:-}" ]; then missing+=("$v"); fi
-          done
-          if [ "${#missing[@]}" -gt 0 ]; then
-            echo "::error::rig lane misconfigured, missing: ${missing[*]}"
+          if ! python -m tools.hardware.preflight --lane destructive --json > preflight.json; then
+            cat preflight.json
+            echo "::error::rig lane preflight failed; see the FAIL checks above"
             exit 1
           fi
 
+      - name: Upload preflight report
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: preflight-destructive-${{ github.run_id }}
+          path: preflight.json
+          if-no-files-found: ignore
+
       - name: Allowlist — refuse any cartridge outside the sacrificial set
         id: allowlist
         run: |
@@ -136,9 +154,6 @@ jobs:
       - name: Gate — fail on any skip or zero passes
         run: python -m tools.hardware.junit_gate reports/*.xml
 
-      # Reboot persistence is a MANUAL step (the runner cannot power-cycle the
-      # appliance): see docs/runbooks/hardware-lanes.md "Reboot persistence".
-
       - name: Recovery report must be clean
         # Only once the allowlist passed: before that, nothing was touched.
         if: always() && steps.allowlist.outcome == 'success'
@@ -176,6 +191,31 @@ jobs:
           python -m tools.hardware.snapshot capture --out snapshots/after.json
           python -m tools.hardware.snapshot diff snapshots/before.json snapshots/after.json --out snapshots/diff.json
 
+      # Soft reboot through the documented AML endpoint. Opt-in per repo: runs
+      # only when vars.HARDWARE_REBOOT_TESTS == 'true' AND every earlier step
+      # succeeded (success(), never always()) — i.e. the tests, the gate, the
+      # recovery report and the lane's after-snapshot (taken first so the lane
+      # diff is not polluted by the reboot; the test diffs its own before/after).
+      # Disruptive but non-destructive to media; same environment approval as
+      # the rest of the job. A hard power-cycle stays manual:
+      # docs/runbooks/hardware-lanes.md "Reboot persistence".
+      - name: Reboot persistence (AML soft reboot)
+        if: ${{ success() && vars.HARDWARE_REBOOT_TESTS == 'true' }}
+        # Bounds the test's own waits (300s down + 600s up) plus snapshots.
+        timeout-minutes: 20
+        env:
+          OPENBLADE_REBOOT_TESTS: enabled
+          # Pinned: this step always verifies TLS, whatever the rig var says.
+          OPENBLADE_APPLIANCE_INSECURE: "false"
+        run: |
+          set -euo pipefail
+          python -m pytest tests/hardware/test_reboot_persistence.py -m real_hardware -q -p no:cacheprovider --tb=short -rs --junitxml=reports/test_reboot_persistence.xml
+          python -m tools.hardware.junit_gate reports/test_reboot_persistence.xml
+
+      - name: Reboot persistence not enabled
+        if: ${{ success() && vars.HARDWARE_REBOOT_TESTS != 'true' }}
+        run: echo "::notice::Reboot persistence skipped — set repo variable HARDWARE_REBOOT_TESTS=true to opt in (docs/runbooks/hardware-lanes.md)."
+
       - name: Upload snapshots
         if: always()
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2

.github/workflows/hardware-library-smoke.yml

@@ -70,21 +70,39 @@ jobs:
 
       - run: pip install -e '.[dev]'
 
+      # Real mode refuses to start without an API token (openblade/config.py
+      # raises RealHardwareDisabledError). Mint a throwaway one per run, masked.
+      - name: Mint a per-run API token
+        run: |
+          set -euo pipefail
+          token=$(openssl rand -hex 32)
+          echo "::add-mask::$token"
+          echo "OPENBLADE_API_TOKEN=$token" >> "$GITHUB_ENV"
+
       # A rig lane must never go green by skipping: the rig tests skip when
       # these are unset, so refuse to run without them. OPENBLADE_CHANGER_DEVICE
       # and OPENBLADE_DRIVE_DEVICES come from the runner host's environment.
-      - name: Preflight — rig configuration present
+      - name: Preflight — rig ready (python -m tools.hardware.preflight)
+        # Read-only tool (env, devices, inquiry serials, changer, allowlist,
+        # appliance login). It must never run armed: blank the job-level enable.
+        env:
+          OPENBLADE_REAL_HARDWARE_ENABLED: ""
         run: |
           set -euo pipefail
-          missing=()
-          for v in OPENBLADE_APPLIANCE_URL OPENBLADE_APPLIANCE_USER OPENBLADE_APPLIANCE_PASSWORD OPENBLADE_CHANGER_DEVICE OPENBLADE_DRIVE_DEVICES; do
-            if [ -z "${!v:-}" ]; then missing+=("$v"); fi
-          done
-          if [ "${#missing[@]}" -gt 0 ]; then
-            echo "::error::rig lane misconfigured, missing: ${missing[*]}"
+          if ! python -m tools.hardware.preflight --lane readonly --json > preflight.json; then
+            cat preflight.json
+            echo "::error::rig lane preflight failed; see the FAIL checks above"
             exit 1
           fi
 
+      - name: Upload preflight report
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: preflight-readonly-${{ github.run_id }}
+          path: preflight.json
+          if-no-files-found: ignore
+
       - name: Snapshot inventory + drive state (before)
         run: |
           set -euo pipefail