diff --git a/.github/workflows/auto-merge-trusted.yml b/.github/workflows/auto-merge-trusted.yml index 62b82ba..47243c2 100644 --- a/.github/workflows/auto-merge-trusted.yml +++ b/.github/workflows/auto-merge-trusted.yml @@ -3,9 +3,15 @@ name: Auto-merge trusted PRs # dangerous-triggers is ignored on `on:` below: this workflow runs only the GitHub API via # github-script on the trusted-author allowlist; never checks out or executes # PR code, which is what makes workflow_run dangerous. +# +# Two required checks live in two workflows (`ci-gate` in CI, `operability-gate` in +# operability), so this listens to BOTH. Whichever finishes first sees the other's gate +# still pending and exits; the second workflow's completion is the retry. After a merge +# it dispatches CI and operability on master, because a merge made with GITHUB_TOKEN +# emits no `push` run (so `full-suite` would never run on the merge commit). on: # zizmor: ignore[dangerous-triggers] -- see comment above workflow_run: - workflows: ["CI"] + workflows: ["CI", "operability"] types: [completed] permissions: @@ -13,28 +19,119 @@ permissions: jobs: merge-trusted-pr: - if: > - github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.event == 'pull_request' + # No conclusion filter here: decide() enforces success per required check, and + # skipping non-success completions would drop the only retry a waiting PR has. + if: github.event.workflow_run.event == 'pull_request' runs-on: ubuntu-latest + # Both workflows can finish together; serialise so the loser sees the PR merged. + concurrency: + group: auto-merge-${{ github.event.workflow_run.head_sha }} + cancel-in-progress: false permissions: contents: write pull-requests: write + checks: read + statuses: read + actions: write steps: - name: Merge trusted PRs targeting master uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: script: | + // decide(requiredContexts, checkRuns, statuses) -> { decision, pending, failed } + // decision is "merge" | "wait" | "fail". Pure: no API calls. + // checkRuns: [{ id, name, status, conclusion }] (all runs on the head SHA) + // statuses: [{ context, state }] (combined status: latest per context) + // A required context counts as passed ONLY on conclusion/state "success". + // The newest check run (highest id) per name wins, so a re-run supersedes. + // A context with no run and no status yet is "pending" (its workflow has + // not reported; that workflow's completion re-triggers this job). + // + // Worked examples (required = ["ci-gate", "operability-gate"]): + // 1. all green: runs [{id:1,name:"ci-gate",status:"completed",conclusion:"success"}, + // {id:2,name:"operability-gate",status:"completed",conclusion:"success"}], statuses [] + // -> { decision: "merge", pending: [], failed: [] } + // 2. one pending (PR #47): runs [{id:1,name:"ci-gate",status:"completed", + // conclusion:"success"}, {id:2,name:"operability-gate",status:"in_progress", + // conclusion:null}] -> { decision: "wait", pending: ["operability-gate"], failed: [] } + // 3. one failed: runs [{id:1,name:"ci-gate",status:"completed",conclusion:"failure"}, + // {id:2,name:"operability-gate",status:"completed",conclusion:"success"}] + // -> { decision: "fail", pending: [], failed: ["ci-gate (failure)"] } + function decide(requiredContexts, checkRuns, statuses) { + const pending = []; + const failed = []; + for (const ctx of requiredContexts) { + const runs = checkRuns.filter((r) => r.name === ctx).sort((a, b) => b.id - a.id); + const status = statuses.find((s) => s.context === ctx); + if (runs.length) { + const run = runs[0]; + if (run.status !== "completed") pending.push(ctx); + else if (run.conclusion !== "success") failed.push(`${ctx} (${run.conclusion})`); + } else if (status) { + if (status.state === "pending") pending.push(ctx); + else if (status.state !== "success") failed.push(`${ctx} (${status.state})`); + } else { + pending.push(ctx); + } + } + const decision = failed.length ? "fail" : pending.length ? "wait" : "merge"; + return { decision, pending, failed }; + } + + // reviewGate(author, reviews, headSha) -> { ok, reason } + // Pure. The owner merges directly. Any other trusted author (Copilot, + // whose PR content is steered by issue text) additionally needs the + // owner's latest substantive review to be APPROVED on the exact head + // SHA; a later push, CHANGES_REQUESTED or DISMISSED review resets it. + const reviewOwner = "Amantux"; + function reviewGate(author, reviews, headSha) { + if (author === reviewOwner) return { ok: true, reason: "owner" }; + let latest = null; + for (const r of reviews) { + if (!r.user || r.user.login !== reviewOwner) continue; + if (r.state === "COMMENTED" || r.state === "PENDING") continue; + latest = r; // listReviews is chronological + } + if (!latest || latest.state !== "APPROVED") { + return { ok: false, reason: `no ${reviewOwner} approval` }; + } + if (latest.commit_id !== headSha) { + return { ok: false, reason: `${reviewOwner} approval is on an older commit` }; + } + return { ok: true, reason: "approved" }; + } + const owner = context.repo.owner; const repo = context.repo.repo; const allowedAuthors = new Set(["Amantux", "Copilot"]); + const fallbackRequired = ["ci-gate", "operability-gate"]; const prs = context.payload.workflow_run.pull_requests || []; if (!prs.length) { - core.info("No pull requests attached to this CI run."); + core.info("No pull requests attached to this workflow run."); return; } + async function requiredContextsFor(branch) { + try { + const { data } = await github.rest.repos.getStatusChecksProtection({ + owner, + repo, + branch, + }); + if (data.contexts && data.contexts.length) return data.contexts; + return fallbackRequired; + } catch (error) { + // GITHUB_TOKEN cannot be granted `administration: read`, so this is + // usually 403/404; fall back to the known branch-protection list. + if (error.status === 403 || error.status === 404) { + core.info(`Branch protection not readable (${error.status}); using ${fallbackRequired.join(", ")}.`); + return fallbackRequired; + } + throw error; + } + } + for (const prRef of prs) { const number = prRef.number; const { data: pr } = await github.rest.pulls.get({ @@ -60,15 +157,77 @@ jobs: continue; } + const required = await requiredContextsFor(pr.base.ref); + // Only GitHub Actions check runs count, so another app cannot post a + // look-alike gate name. + const checkRuns = ( + await github.paginate(github.rest.checks.listForRef, { + owner, + repo, + ref: pr.head.sha, + per_page: 100, + }) + ).filter((r) => r.app && r.app.slug === "github-actions"); + const { data: combined } = await github.rest.repos.getCombinedStatusForRef({ + owner, + repo, + ref: pr.head.sha, + per_page: 100, + }); + const verdict = decide(required, checkRuns, combined.statuses || []); + if (verdict.decision === "wait") { + core.notice(`Not merging #${number} yet: waiting for ${verdict.pending.join(", ")}; the other workflow's completion will re-run this.`); + continue; + } + if (verdict.decision === "fail") { + core.notice(`Not merging #${number}: required check(s) not successful: ${verdict.failed.join(", ")}.`); + continue; + } + + const reviews = await github.paginate(github.rest.pulls.listReviews, { + owner, + repo, + pull_number: number, + per_page: 100, + }); + const review = reviewGate(pr.user.login, reviews, pr.head.sha); + if (!review.ok) { + core.notice(`Not merging #${number}: ${review.reason} on ${pr.head.sha}.`); + continue; + } + try { await github.rest.pulls.merge({ owner, repo, pull_number: number, merge_method: "squash", + sha: pr.head.sha, }); core.info(`Merged trusted PR #${number}.`); } catch (error) { core.warning(`Unable to merge #${number}: ${error.message}`); + continue; + } + + // A GITHUB_TOKEN merge emits no `push` run; workflow_dispatch is the one + // event this token can trigger, so run the master-only lanes explicitly. + const dispatchFailures = []; + for (const workflowId of ["ci.yml", "operability.yml"]) { + try { + await github.rest.actions.createWorkflowDispatch({ + owner, + repo, + workflow_id: workflowId, + ref: "master", + }); + core.info(`Dispatched ${workflowId} on master after merging #${number}.`); + } catch (error) { + dispatchFailures.push(`${workflowId}: ${error.message}`); + } + } + if (dispatchFailures.length) { + // Fail red: without these runs full-suite never covers the merge. + core.setFailed(`Merged #${number} but could not dispatch master lanes: ${dispatchFailures.join("; ")}`); } } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd986d0..2d72ec2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,9 @@ on: concurrency: group: ci-${{ github.ref }} - cancel-in-progress: true + # PR runs supersede each other; dispatched master runs (post-merge full-suite / + # operability) must never cancel one another or a merge ends up with no result. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read diff --git a/.github/workflows/hardware-destructive.yml b/.github/workflows/hardware-destructive.yml index 6a1902f..d3b4dc8 100644 --- a/.github/workflows/hardware-destructive.yml +++ b/.github/workflows/hardware-destructive.yml @@ -83,21 +83,39 @@ jobs: - run: pip install -e '.[dev]' + # Real mode refuses to start without an API token (openblade/config.py + # raises RealHardwareDisabledError). Mint a throwaway one per run, masked. + - name: Mint a per-run API token + run: | + set -euo pipefail + token=$(openssl rand -hex 32) + echo "::add-mask::$token" + echo "OPENBLADE_API_TOKEN=$token" >> "$GITHUB_ENV" + # A rig lane must never go green by skipping: the rig tests skip when # these are unset, so refuse to run without them. OPENBLADE_CHANGER_DEVICE # and OPENBLADE_DRIVE_DEVICES come from the runner host's environment. - - name: Preflight — rig configuration present + - name: Preflight — rig ready (python -m tools.hardware.preflight) + # Read-only tool (env, devices, inquiry serials, changer, allowlist, + # appliance login). It must never run armed: blank the job-level enable. + env: + OPENBLADE_REAL_HARDWARE_ENABLED: "" run: | set -euo pipefail - missing=() - for v in OPENBLADE_APPLIANCE_URL OPENBLADE_APPLIANCE_USER OPENBLADE_APPLIANCE_PASSWORD OPENBLADE_CHANGER_DEVICE OPENBLADE_DRIVE_DEVICES; do - if [ -z "${!v:-}" ]; then missing+=("$v"); fi - done - if [ "${#missing[@]}" -gt 0 ]; then - echo "::error::rig lane misconfigured, missing: ${missing[*]}" + if ! python -m tools.hardware.preflight --lane destructive --json > preflight.json; then + cat preflight.json + echo "::error::rig lane preflight failed; see the FAIL checks above" exit 1 fi + - name: Upload preflight report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: preflight-destructive-${{ github.run_id }} + path: preflight.json + if-no-files-found: ignore + - name: Allowlist — refuse any cartridge outside the sacrificial set id: allowlist run: | @@ -136,9 +154,6 @@ jobs: - name: Gate — fail on any skip or zero passes run: python -m tools.hardware.junit_gate reports/*.xml - # Reboot persistence is a MANUAL step (the runner cannot power-cycle the - # appliance): see docs/runbooks/hardware-lanes.md "Reboot persistence". - - name: Recovery report must be clean # Only once the allowlist passed: before that, nothing was touched. if: always() && steps.allowlist.outcome == 'success' @@ -176,6 +191,31 @@ jobs: python -m tools.hardware.snapshot capture --out snapshots/after.json python -m tools.hardware.snapshot diff snapshots/before.json snapshots/after.json --out snapshots/diff.json + # Soft reboot through the documented AML endpoint. Opt-in per repo: runs + # only when vars.HARDWARE_REBOOT_TESTS == 'true' AND every earlier step + # succeeded (success(), never always()) — i.e. the tests, the gate, the + # recovery report and the lane's after-snapshot (taken first so the lane + # diff is not polluted by the reboot; the test diffs its own before/after). + # Disruptive but non-destructive to media; same environment approval as + # the rest of the job. A hard power-cycle stays manual: + # docs/runbooks/hardware-lanes.md "Reboot persistence". + - name: Reboot persistence (AML soft reboot) + if: ${{ success() && vars.HARDWARE_REBOOT_TESTS == 'true' }} + # Bounds the test's own waits (300s down + 600s up) plus snapshots. + timeout-minutes: 20 + env: + OPENBLADE_REBOOT_TESTS: enabled + # Pinned: this step always verifies TLS, whatever the rig var says. + OPENBLADE_APPLIANCE_INSECURE: "false" + run: | + set -euo pipefail + python -m pytest tests/hardware/test_reboot_persistence.py -m real_hardware -q -p no:cacheprovider --tb=short -rs --junitxml=reports/test_reboot_persistence.xml + python -m tools.hardware.junit_gate reports/test_reboot_persistence.xml + + - name: Reboot persistence not enabled + if: ${{ success() && vars.HARDWARE_REBOOT_TESTS != 'true' }} + run: echo "::notice::Reboot persistence skipped — set repo variable HARDWARE_REBOOT_TESTS=true to opt in (docs/runbooks/hardware-lanes.md)." + - name: Upload snapshots if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 diff --git a/.github/workflows/hardware-library-smoke.yml b/.github/workflows/hardware-library-smoke.yml index 5301d21..891f024 100644 --- a/.github/workflows/hardware-library-smoke.yml +++ b/.github/workflows/hardware-library-smoke.yml @@ -70,21 +70,39 @@ jobs: - run: pip install -e '.[dev]' + # Real mode refuses to start without an API token (openblade/config.py + # raises RealHardwareDisabledError). Mint a throwaway one per run, masked. + - name: Mint a per-run API token + run: | + set -euo pipefail + token=$(openssl rand -hex 32) + echo "::add-mask::$token" + echo "OPENBLADE_API_TOKEN=$token" >> "$GITHUB_ENV" + # A rig lane must never go green by skipping: the rig tests skip when # these are unset, so refuse to run without them. OPENBLADE_CHANGER_DEVICE # and OPENBLADE_DRIVE_DEVICES come from the runner host's environment. - - name: Preflight — rig configuration present + - name: Preflight — rig ready (python -m tools.hardware.preflight) + # Read-only tool (env, devices, inquiry serials, changer, allowlist, + # appliance login). It must never run armed: blank the job-level enable. + env: + OPENBLADE_REAL_HARDWARE_ENABLED: "" run: | set -euo pipefail - missing=() - for v in OPENBLADE_APPLIANCE_URL OPENBLADE_APPLIANCE_USER OPENBLADE_APPLIANCE_PASSWORD OPENBLADE_CHANGER_DEVICE OPENBLADE_DRIVE_DEVICES; do - if [ -z "${!v:-}" ]; then missing+=("$v"); fi - done - if [ "${#missing[@]}" -gt 0 ]; then - echo "::error::rig lane misconfigured, missing: ${missing[*]}" + if ! python -m tools.hardware.preflight --lane readonly --json > preflight.json; then + cat preflight.json + echo "::error::rig lane preflight failed; see the FAIL checks above" exit 1 fi + - name: Upload preflight report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: preflight-readonly-${{ github.run_id }} + path: preflight.json + if-no-files-found: ignore + - name: Snapshot inventory + drive state (before) run: | set -euo pipefail diff --git a/.github/workflows/hardware-nightly-readonly.yml b/.github/workflows/hardware-nightly-readonly.yml index 413020b..ff96d13 100644 --- a/.github/workflows/hardware-nightly-readonly.yml +++ b/.github/workflows/hardware-nightly-readonly.yml @@ -69,21 +69,39 @@ jobs: - run: pip install -e '.[dev]' + # Real mode refuses to start without an API token (openblade/config.py + # raises RealHardwareDisabledError). Mint a throwaway one per run, masked. + - name: Mint a per-run API token + run: | + set -euo pipefail + token=$(openssl rand -hex 32) + echo "::add-mask::$token" + echo "OPENBLADE_API_TOKEN=$token" >> "$GITHUB_ENV" + # A rig lane must never go green by skipping: the rig tests skip when # these are unset, so refuse to run without them. OPENBLADE_CHANGER_DEVICE # and OPENBLADE_DRIVE_DEVICES come from the runner host's environment. - - name: Preflight — rig configuration present + - name: Preflight — rig ready (python -m tools.hardware.preflight) + # Read-only tool (env, devices, inquiry serials, changer, allowlist, + # appliance login). It must never run armed: blank the job-level enable. + env: + OPENBLADE_REAL_HARDWARE_ENABLED: "" run: | set -euo pipefail - missing=() - for v in OPENBLADE_APPLIANCE_URL OPENBLADE_APPLIANCE_USER OPENBLADE_APPLIANCE_PASSWORD OPENBLADE_CHANGER_DEVICE OPENBLADE_DRIVE_DEVICES; do - if [ -z "${!v:-}" ]; then missing+=("$v"); fi - done - if [ "${#missing[@]}" -gt 0 ]; then - echo "::error::rig lane misconfigured, missing: ${missing[*]}" + if ! python -m tools.hardware.preflight --lane readonly --json > preflight.json; then + cat preflight.json + echo "::error::rig lane preflight failed; see the FAIL checks above" exit 1 fi + - name: Upload preflight report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: preflight-readonly-${{ github.run_id }} + path: preflight.json + if-no-files-found: ignore + # No allowlist step: this lane names no cartridges and performs no # motion. The snapshot diff below is what would catch a violation. - name: Snapshot inventory + drive state (before) diff --git a/.github/workflows/mhvtl-weekly.yml b/.github/workflows/mhvtl-weekly.yml index 97b46c1..ba1ee83 100644 --- a/.github/workflows/mhvtl-weekly.yml +++ b/.github/workflows/mhvtl-weekly.yml @@ -283,6 +283,15 @@ jobs: sudo dmesg | tail -80 || true cat /etc/mhvtl/device.conf || true + # Real mode refuses to start without an API token (openblade/config.py + # raises RealHardwareDisabledError). Mint a throwaway one per run, masked. + - name: Mint a per-run API token + run: | + set -euo pipefail + token=$(openssl rand -hex 32) + echo "::add-mask::$token" + echo "OPENBLADE_API_TOKEN=$token" >> "$GITHUB_ENV" + # Resolving the rig is its own step so that "the rig cannot be addressed" # fails HERE, loudly, instead of leaking into the pytest step. # @@ -329,6 +338,7 @@ jobs: sudo env \ OPENBLADE_BACKEND="$OPENBLADE_BACKEND" \ OPENBLADE_REAL_HARDWARE_ENABLED="$OPENBLADE_REAL_HARDWARE_ENABLED" \ + OPENBLADE_API_TOKEN="$OPENBLADE_API_TOKEN" \ OPENBLADE_CHANGER_DEVICE="$OPENBLADE_CHANGER_DEVICE" \ OPENBLADE_DRIVE_DEVICES="$OPENBLADE_DRIVE_DEVICES" \ "$(command -v python)" -m tools.hardware.snapshot capture --out snapshots/before.json @@ -346,6 +356,7 @@ jobs: # and if it somehow did not, say which one is missing. : "${OPENBLADE_BACKEND:?not set — the rig-env step did not run}" : "${OPENBLADE_REAL_HARDWARE_ENABLED:?not set — the rig-env step did not run}" + : "${OPENBLADE_API_TOKEN:?not set — the mint-token step did not run}" : "${OPENBLADE_CHANGER_DEVICE:?not set — the rig-env step did not run}" : "${OPENBLADE_DRIVE_DEVICES:?not set — the rig-env step did not run}" : "${OPENBLADE_SCRATCH_BARCODES:?not set — the rig-env step did not run}" @@ -376,6 +387,7 @@ jobs: sudo env \ OPENBLADE_BACKEND="$OPENBLADE_BACKEND" \ OPENBLADE_REAL_HARDWARE_ENABLED="$OPENBLADE_REAL_HARDWARE_ENABLED" \ + OPENBLADE_API_TOKEN="$OPENBLADE_API_TOKEN" \ OPENBLADE_CHANGER_DEVICE="$OPENBLADE_CHANGER_DEVICE" \ OPENBLADE_DRIVE_DEVICES="$OPENBLADE_DRIVE_DEVICES" \ OPENBLADE_SCRATCH_BARCODES="$OPENBLADE_SCRATCH_BARCODES" \ @@ -403,6 +415,7 @@ jobs: sudo env \ OPENBLADE_BACKEND="$OPENBLADE_BACKEND" \ OPENBLADE_REAL_HARDWARE_ENABLED="$OPENBLADE_REAL_HARDWARE_ENABLED" \ + OPENBLADE_API_TOKEN="$OPENBLADE_API_TOKEN" \ OPENBLADE_CHANGER_DEVICE="$OPENBLADE_CHANGER_DEVICE" \ OPENBLADE_DRIVE_DEVICES="$OPENBLADE_DRIVE_DEVICES" \ "$(command -v python)" -m tools.hardware.snapshot capture --out snapshots/after.json diff --git a/.github/workflows/operability.yml b/.github/workflows/operability.yml index f883731..59249a9 100644 --- a/.github/workflows/operability.yml +++ b/.github/workflows/operability.yml @@ -17,7 +17,9 @@ on: concurrency: group: operability-${{ github.ref }} - cancel-in-progress: true + # PR runs supersede each other; dispatched master runs (post-merge full-suite / + # operability) must never cancel one another or a merge ends up with no result. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} permissions: contents: read @@ -77,5 +79,7 @@ jobs: OPENBLADE_ADMIN_PASSWORD: ci-strong-admin OPENBLADE_SERVICE_PASSWORD: ci-strong-service OPENBLADE_SERVICE_TOKEN: ci-strong-token + # Production refuses to start with the native API open (native_api_open). + OPENBLADE_API_TOKEN: ci-strong-api-token OPENBLADE_DB_URL: sqlite:////tmp/openblade-ci-deploy.db run: python scripts/deploy.py --skip-deploy diff --git a/AGENTS.md b/AGENTS.md index 23662ce..3082f38 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,6 +7,10 @@ Develop and maintain OpenBlade, a high-reliability, simulator-first tape archive ### Hardware Safety & Data Integrity - **Safety Override:** Never bypass or disable the real-hardware safety guard mechanisms. - **Destructive Operations:** Never perform format or erase operations without positive barcode confirmation and a cryptographically valid safety token. +- **Single-use format token:** `FormatService.confirm` atomically consumes the safety token BEFORE the job; a reused, concurrent, or failed-delete token is refused and a failed job never re-arms it. +- **Scratch-media allowlist in code:** on a real backend `FormatService` formats only barcodes in `OPENBLADE_SCRATCH_BARCODES` (`openblade/domain/media_allowlist.py`); an empty allowlist refuses everything. +- **No ingest auto-format on real hardware:** NAS ingest never formats on a real backend (it fails the job instead); formatting is only the two-phase `/ltfs/format` flow. +- **Third real-mode signal:** `OPENBLADE_BACKEND=real` + `OPENBLADE_REAL_HARDWARE_ENABLED=true` also requires `OPENBLADE_API_TOKEN` (or `_FILE`), else `load_config` raises `RealHardwareDisabledError`; `simulator` is an alias of `mock`. - **State Management:** Never initiate an unload sequence while the Linear Tape File System (LTFS) is mounted or in a "dirty" state. - **Verification:** Every hardware command must be preceded by a state-check and followed by a verification of the resulting state. diff --git a/Makefile b/Makefile index 0c792eb..4b50645 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL := /bin/bash .ONESHELL: -.PHONY: mutation up down logs emulator-up emulator-down emulator-logs emulator-config emulator-ps fleet-up fleet-down fleet-logs fleet-ps fleet-config fleet-build up-standalone test test-unit test-integration test-contract seed-libraries seed-tapes seed-all clean build-frontend dev-backend dev-frontend dev frontend lint build all test-hardware protocols-up protocols-down test-protocols +.PHONY: mutation up down logs emulator-up emulator-down emulator-logs emulator-config emulator-ps fleet-up fleet-down fleet-logs fleet-ps fleet-config fleet-build up-standalone test test-unit test-integration test-contract seed-libraries seed-tapes seed-all clean build-frontend dev-backend dev-frontend dev frontend lint build all test-hardware protocols-up protocols-down test-protocols hw-preflight lan-up lan-down # Start all services up: @@ -112,6 +112,10 @@ all: lint test build test-hardware: OPENBLADE_BACKEND=real OPENBLADE_REAL_HARDWARE_ENABLED=true python3 -m pytest tests/hardware/ -v +# Read-only day-one rig check (no motion; must not run armed). Lane: PREFLIGHT_LANE=readonly|destructive. +hw-preflight: + python3 -m tools.hardware.preflight --lane $${PREFLIGHT_LANE:-readonly} + # NAS protocol rig (SMB/NFS, containerised). Host ports: OB_SMB_PORT / OB_NFS_PORT. PROTOCOLS_COMPOSE = docker compose -f deploy/nas-protocols/docker-compose.yml @@ -128,3 +132,13 @@ test-protocols: # Mutation lane (nightly); see tools/mutation_run.sh and mutation/baseline.txt mutation: PYTHON=$${PYTHON:-python} MUTATION_TIMEOUT=$${MUTATION_TIMEOUT:-3600} tools/mutation_run.sh + +# LAN exposure (production mode, ports 8090/8091/8092). Needs deploy/lan/.env +# filled from deploy/lan/.env.example; see deploy/lan/README.md. +LAN_COMPOSE = docker compose -p openblade-lan -f docker-compose.yml -f deploy/lan/docker-compose.lan.yml --profile web-flask --env-file deploy/lan/.env + +lan-up: + $(LAN_COMPOSE) up -d --build + +lan-down: + $(LAN_COMPOSE) down diff --git a/README.md b/README.md index fd7630c..a89dbe6 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ Backend contract suite: `make test-contract` runs `tests/contract/` against ever **Workflow hygiene (`workflow-lint.yml`).** This workflow runs `actionlint` (with shellcheck on inline `run:` blocks), `zizmor` and `yamllint` (`.yamllint`). Every action is pinned to a full commit SHA with a `# vX.Y.Z` comment. Workflows declare `permissions: contents: read` at the top level and grant write scopes per job. +**Trusted auto-merge (`auto-merge-trusted.yml`).** This workflow squash-merges open, non-draft PRs into `master` when the author is on its trusted list. `Amantux` PRs merge directly; any other trusted author (`Copilot`, whose PR content can be steered by issue text) also needs `Amantux`'s latest review to be `APPROVED` on the PR's current head commit, so a push after approval, or a later "changes requested", holds the merge until it is re-approved. The two required checks come from two workflows: `ci-gate` from `CI` and `operability-gate` from `operability`. So the auto-merge runs when either workflow completes. Before merging, it tries to read the required checks from branch protection. `GITHUB_TOKEN` normally cannot read branch protection, so in practice it uses the fixed list `ci-gate` and `operability-gate`. Update that list in the workflow if the required checks change. It merges only when every required check on the PR head commit reports `success`, and it counts only check runs posted by GitHub Actions. If a check is still running, the run logs a notice and exits, and the other workflow's completion triggers the next attempt. If a check failed, it logs a notice and does not merge. A merge made with `GITHUB_TOKEN` creates no `push` run, so after merging the workflow dispatches `ci.yml` and `operability.yml` on `master`. That dispatch runs `full-suite` on the current `master` tip. If two merges land close together, the concurrency group cancels the first run, so only the later commit is covered. If a dispatch fails, the auto-merge run fails red. + **Nightly (`nightly.yml`).** Runs on a cron schedule and through `workflow_dispatch`. It is not a required PR check, but each of its lanes fails red: | Lane | Selection | diff --git a/deploy/lan/.env.example b/deploy/lan/.env.example new file mode 100644 index 0000000..3823a19 --- /dev/null +++ b/deploy/lan/.env.example @@ -0,0 +1,20 @@ +# Copy to deploy/lan/.env (gitignored by .env*) and fill EVERY value. +# Generate each secret with: +# python -c "import secrets;print(secrets.token_urlsafe(32))" + +# Interface to publish on (0.0.0.0 = all; set to the LAN NIC IP to narrow it). +OPENBLADE_BIND_IP=0.0.0.0 +# Bearer token for every OpenBlade-native route (Authorization: Bearer ). +OPENBLADE_API_TOKEN= +# Controller header token (X-Openblade-Service-Token) for /aml mount/unmount. +OPENBLADE_SERVICE_TOKEN= +# Emulated i3 AML admin / service account passwords (defaults are refused). +OPENBLADE_ADMIN_PASSWORD= +OPENBLADE_SERVICE_PASSWORD= +# Flask session signing key for the web-flask UI (port 8092). +FLASK_SECRET_KEY= +# Explicit origins only ('*' is refused). The SPA is same-origin via nginx, so +# this only matters for other browser clients, e.g. http://:8091 +OPENBLADE_CORS_ORIGINS= +# Leave false on plain HTTP; set true once TLS terminates in front. +OPENBLADE_WEB_SECURE_COOKIES=false diff --git a/deploy/lan/README.md b/deploy/lan/README.md new file mode 100644 index 0000000..dca8245 --- /dev/null +++ b/deploy/lan/README.md @@ -0,0 +1,31 @@ +# Exposing OpenBlade on a LAN + +The default `docker-compose.yml` binds every port to `127.0.0.1`. This override +publishes the stack on the LAN in **production mode** (fail-closed auth). + +| Service | LAN port | Container | +|------------|----------|-----------| +| api | 8090 | 8000 | +| web (SPA) | 8091 | 80 | +| web-flask | 8092 | 8080 | + +## Steps + +1. `cp deploy/lan/.env.example deploy/lan/.env` (gitignored) and fill every value. + Generate each secret with + `python -c "import secrets;print(secrets.token_urlsafe(32))"`. +2. `make lan-up` (build + start, project `openblade-lan`); `make lan-down` to stop. + +Compose aborts naming the variable if any required secret is unset, and the api +refuses to start in production with an unset API token, a default AML +admin/service password or service token, or `*` in `OPENBLADE_CORS_ORIGINS`. + +## Notes + +- The SPA's nginx proxies `/api` and `/aml` to `api:8000`, so the browser only + talks to port 8091 (same origin) and CORS does not apply to it. Set + `OPENBLADE_CORS_ORIGINS` to explicit origins for any other browser client. +- The test-runner route is disabled (404) in production. +- **HTTP on a LAN is cleartext**: tokens and passwords cross the wire in the + clear. Terminate TLS in a reverse proxy next, then set + `OPENBLADE_WEB_SECURE_COOKIES=true`. diff --git a/deploy/lan/docker-compose.lan.yml b/deploy/lan/docker-compose.lan.yml new file mode 100644 index 0000000..7445e81 --- /dev/null +++ b/deploy/lan/docker-compose.lan.yml @@ -0,0 +1,40 @@ +# LAN exposure override. Use ONLY via `make lan-up` / `make lan-down`: +# docker compose -p openblade-lan -f docker-compose.yml \ +# -f deploy/lan/docker-compose.lan.yml --profile web-flask \ +# --env-file deploy/lan/.env up -d --build +# Publishes api/web/web-flask on ${OPENBLADE_BIND_IP:-0.0.0.0}:8090/8091/8092 +# (8000/5173/5175 belong to other projects on the target box) and runs in +# production mode, so the api REFUSES TO START with any open auth gate. Every +# secret below is required: compose aborts with a named error if one is unset. +# HTTP on a LAN is cleartext -- put a TLS reverse proxy in front next. +name: openblade-lan + +services: + api: + ports: !override + - "${OPENBLADE_BIND_IP:-0.0.0.0}:8090:8000" + environment: + OPENBLADE_ENV: production + OPENBLADE_API_TOKEN: ${OPENBLADE_API_TOKEN:?set OPENBLADE_API_TOKEN in deploy/lan/.env} + OPENBLADE_SERVICE_TOKEN: ${OPENBLADE_SERVICE_TOKEN:?set OPENBLADE_SERVICE_TOKEN in deploy/lan/.env} + OPENBLADE_ADMIN_PASSWORD: ${OPENBLADE_ADMIN_PASSWORD:?set OPENBLADE_ADMIN_PASSWORD in deploy/lan/.env} + OPENBLADE_SERVICE_PASSWORD: ${OPENBLADE_SERVICE_PASSWORD:?set OPENBLADE_SERVICE_PASSWORD in deploy/lan/.env} + OPENBLADE_CORS_ORIGINS: ${OPENBLADE_CORS_ORIGINS:?set OPENBLADE_CORS_ORIGINS in deploy/lan/.env} + + web: + # nginx proxies /api and /aml to api:8000 by compose name, so the browser + # only talks to this port (same origin); CORS does not apply to the SPA. + ports: !override + - "${OPENBLADE_BIND_IP:-0.0.0.0}:8091:80" + + web-flask: + ports: !override + - "${OPENBLADE_BIND_IP:-0.0.0.0}:8092:8080" + environment: + OPENBLADE_ENV: production + FLASK_SECRET_KEY: ${FLASK_SECRET_KEY:?set FLASK_SECRET_KEY in deploy/lan/.env} + OPENBLADE_API_TOKEN: ${OPENBLADE_API_TOKEN:?set OPENBLADE_API_TOKEN in deploy/lan/.env} + OPENBLADE_SERVICE_TOKEN: ${OPENBLADE_SERVICE_TOKEN:?set OPENBLADE_SERVICE_TOKEN in deploy/lan/.env} + # Production defaults cookies to Secure, which browsers drop over plain + # HTTP (login would silently fail). Set true once a TLS proxy is in front. + OPENBLADE_WEB_SECURE_COOKIES: ${OPENBLADE_WEB_SECURE_COOKIES:-false} diff --git a/docker-compose.yml b/docker-compose.yml index 7e97d47..720e75b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: volumes: - openblade-data:/data ports: - - "8000:8000" + - "127.0.0.1:8000:8000" extra_hosts: - "host.docker.internal:host-gateway" healthcheck: @@ -34,14 +34,14 @@ services: image: ${OPENBLADE_EMULATOR_IMAGE:-ghcr.io/amantux/openblade-i3-emulator:0.1.0} profiles: ["emulator"] ports: - - "8010:8010" + - "127.0.0.1:8010:8010" web: build: context: ./frontend dockerfile: Dockerfile ports: - - "5173:80" + - "127.0.0.1:5173:80" depends_on: api: condition: service_healthy @@ -101,7 +101,7 @@ services: # literal here keeps the two in step; exporting the var overrides both. OPENBLADE_SERVICE_TOKEN: ${OPENBLADE_SERVICE_TOKEN:-openblade-controller-dev-token-do-not-expose} ports: - - "5175:8080" + - "127.0.0.1:5175:8080" depends_on: api: condition: service_healthy diff --git a/docs/decisions/2026-10-09-persistent-drive-leases.md b/docs/decisions/2026-10-09-persistent-drive-leases.md index 1273934..4317ee0 100644 --- a/docs/decisions/2026-10-09-persistent-drive-leases.md +++ b/docs/decisions/2026-10-09-persistent-drive-leases.md @@ -311,8 +311,18 @@ Decisions (WS-capacity, fleet round 3): with a message, CRITICAL is DEGRADED, never UNHEALTHY. Thresholds are managed by `GET`/`PUT /nas/scratch-thresholds` (404 unset, 422 on `warn < min`). -Reviewer risks still open: `book_plan_capacity` books against `plan.pool`, not the -`pool_id` of the start request, so a plan previewed for one pool and started into -another is budgeted against the first; the share-push registers its plan before -booking, so a 409 there leaves a registered plan with no job; and a started job that -never reaches a terminal state without raising holds its bytes for the full hour. +Reviewer risks, resolved (2026-10-11): +- **Pool mismatch — resolved.** `POST /nas/ingest/start` returns 409 (`pool_id does + not match plan pool`) when `request.pool_id != plan.pool`, including a pool-less + preview started into a pool; the job's dataset is created in `plan.pool`, the pool + that was booked. +- **Orphan plan on share-push 409 — resolved.** Share-push books first and registers + the plan only after the booking succeeds. +- **Booking held after a failed start — resolved.** In both start paths everything + after `book_plan_capacity` runs under try/except: on any exception the booking is + released (`release_plan_capacity`, owner = `plan_id`) and the plan is dropped + (`unregister_archive_plan`), then the original exception is re-raised. A failed + release is logged and left to the TTL; it never masks the start error. + +Still open: a started job that never reaches a terminal state without raising holds +its bytes for the full hour. diff --git a/docs/deployment.md b/docs/deployment.md index 3b5307a..fda2171 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -106,3 +106,29 @@ deploy fails the build. No production data or credentials are used. yet been exercised on a real tag**. - `rollback` redeploys application code only. It never touches tape state; if the catalog needs restoring, follow docs/disaster-recovery.md. + +## Exposing on a LAN + +`docker-compose.yml` binds every published port to `127.0.0.1`. To reach the +stack from other machines use the production override in `deploy/lan/` +(full steps in `deploy/lan/README.md`): + +- **Ports:** api `8090`, React UI `8091`, Flask UI `8092`, on + `${OPENBLADE_BIND_IP:-0.0.0.0}`. The React UI proxies `/api` and `/aml` to + the api by compose name, so browsers only need port 8091. +- **Required secrets** (compose refuses to render without them): + `OPENBLADE_API_TOKEN`, `OPENBLADE_SERVICE_TOKEN`, `OPENBLADE_ADMIN_PASSWORD`, + `OPENBLADE_SERVICE_PASSWORD`, `FLASK_SECRET_KEY`, `OPENBLADE_CORS_ORIGINS`. + Generate each with + `python -c "import secrets;print(secrets.token_urlsafe(32))"`. +- **Fail-closed:** with `OPENBLADE_ENV=production` the api refuses to start + (`ConfigurationError`) on an unset API token, a default AML admin/service + password or service token, or `*` in CORS origins; the Flask UI refuses to + start without `FLASK_SECRET_KEY`; the test-runner route returns 404. + Outside production the api logs one banner listing every open gate, and + `scripts/validate_config.py` reports the same set. +- **Cleartext:** plain HTTP on a LAN exposes tokens and passwords to anyone on + the segment. Terminate TLS in a reverse proxy next, then set + `OPENBLADE_WEB_SECURE_COOKIES=true` (the override defaults it to `false` + because Secure cookies are dropped over HTTP). +- `make lan-up` / `make lan-down` start and stop the `openblade-lan` project. diff --git a/docs/runbooks/hardware-lanes.md b/docs/runbooks/hardware-lanes.md index 4adf692..53ccf1b 100644 --- a/docs/runbooks/hardware-lanes.md +++ b/docs/runbooks/hardware-lanes.md @@ -24,6 +24,47 @@ refuses (exit 2) if the backend is in hardware dry-run mode. and job id (`register-and-validate`) keep resolving for any branch-protection rule that references them. +## 0. Day one (in order) + +Do these once, in this order. Each step says what "green" looks like; do not +move on until it is. + +1. **Register the runner** with label `i3` (§1). Green: the runner shows + *Idle* under Settings → Actions → Runners with labels `self-hosted, i3`. +2. **Write the runner `.env`**: `OPENBLADE_CHANGER_DEVICE=/dev/sgN`, + `OPENBLADE_DRIVE_DEVICES=/dev/nstN,...` (DTE order), optional + `OPENBLADE_DRIVE_SERIAL_MAP=:,...`; runner user in the `tape` + and `disk` groups; `sg3-utils`, `lsscsi`, `mtx`, LTFS installed. Green: the + runner service restarts and `id` lists both groups. +3. **Set repo variables and secrets** (§2) — exact names: + + | Kind | Name | Purpose | + |---|---|---| + | var | `HARDWARE_RIG_AVAILABLE` | `true` enables the rig lanes (set LAST) | + | var | `SACRIFICIAL_BARCODES` | scratch cartridges the destructive lane may touch | + | var | `OPENBLADE_APPLIANCE_URL` | `https://` | + | var | `OPENBLADE_APPLIANCE_USER` | AML user for the read-only API checks | + | var | `OPENBLADE_APPLIANCE_INSECURE` | `true` only for a self-signed cert (preflight WARNs) | + | secret | `OPENBLADE_APPLIANCE_PASSWORD` | password for that user | + | secret | `QUANTUM_AML_USER` / `QUANTUM_AML_PASSWORD` | AML login (→ `I3_AML_USER/PASSWORD`) | + + Green: all names appear under Settings → Secrets and variables → Actions. +4. **Create the `hardware-destructive` environment with required reviewers** + (§3). Green: the environment lists at least one reviewer. +5. **Run `make hw-preflight` on the runner host, as the runner user**, with the + lane env exported (add `PREFLIGHT_LANE=destructive` once the allowlist is + set). It is read-only (inquiry/listing and an appliance login/logout only) + and must not run with `OPENBLADE_REAL_HARDWARE_ENABLED` set. Green: exit 0, + no `FAIL` lines; every `WARN` is understood (e.g. insecure TLS). +6. **Dispatch `hardware-library-smoke`**. Green: the job passes, its JUnit gate + reports zero skips, and the `preflight-readonly-*` artifact shows `"ok": true`. +7. **Capture the compatibility evidence**: + `python tools/compat_capture.py capture … --source captured --attest ""` + (full command in `compatibility/README.md`). Green: the captured files are + committed and the attestation names the operator and firmware. +8. **Let the nightly read-only lane watch itself.** Enable the destructive lane + only after one green smoke run AND one green read-only night. + ## 1. Register the self-hosted runner 1. On the host attached to the i3 (SAS/FC to the changer and drives), create an @@ -39,6 +80,11 @@ rule that references them. 4. Install host tools the suites call: `sg3-utils`, `lsscsi`, `mtx`, LTFS. 5. Add repo secrets `QUANTUM_AML_USER` / `QUANTUM_AML_PASSWORD` (AML login). +The lanes mint a per-run `OPENBLADE_API_TOKEN`; real mode refuses to start without one, +so do not set a token in the runner's `.env`. + +Verify the host with `make hw-preflight` (§0 step 5) before dispatching any lane. + ## 2. Repository variables Repo → **Settings → Secrets and variables → Actions → Variables**: @@ -59,6 +105,12 @@ Repo → **Settings → Secrets and variables → Actions → Variables**: writes JUnit XML and a final **Gate** step (`python -m tools.hardware.junit_gate`) fails the job if any test skipped or nothing passed. An expected skip must be passed explicitly (`--expected-skip classname::name`) with a comment saying why. +- **`HARDWARE_REBOOT_TESTS`** = `true` — optional opt-in for the destructive + lane's soft-reboot step (see §5 "Reboot persistence"). Unset (the default), + the step is skipped and the run shows a `::notice::` saying so. + +Every rig lane starts with `python -m tools.hardware.preflight --lane --json`; +a missing variable shows up there as a `FAIL` naming it (artifact `preflight--`). ## 3. Environment approval (settings-only — cannot be committed) @@ -71,6 +123,9 @@ Repo → **Settings → Environments → New environment** `hardware-destructive Without the environment, GitHub creates it on first use with no protection — so create it, with reviewers, before setting `HARDWARE_RIG_AVAILABLE`. +The destructive lane's preflight (`--lane destructive`) additionally FAILs if +`SACRIFICIAL_BARCODES` is empty, names a cartridge not in inventory, or one is loaded. + ## 4. The cartridge allowlist Before any hardware step, the destructive lane (and its mhvtl rehearsal) runs: @@ -90,6 +145,10 @@ Each lane also captures inventory + drive state before and after (`tools.hardware.snapshot capture`), diffs them, and uploads `snapshots/` as an artifact even on failure. On the read-only lanes the diff must be empty. +The allowlist is also enforced by the service itself: on a real backend +`FormatService` refuses to dry-run or confirm a format for any barcode outside +`OPENBLADE_SCRATCH_BARCODES`, and an empty list refuses everything. + ## 5. The destructive sequence Load/unload → LTFS write/read/checksum → archive + restore → sharded write + @@ -100,12 +159,42 @@ LTFS format goes only through the two-phase flow (dry-run plan → explicit barcode + one-time safety token). `test_mkltfs_formats_tape` is deselected in this lane because it calls `mkltfs` directly. -### Reboot persistence (manual) - -The runner cannot power-cycle the appliance. Once per quarter, after a green -destructive run: reboot the i3 from its web UI, wait for it to come Ready, -dispatch `hardware-library-smoke.yml`, and confirm the snapshot diff against -the pre-reboot `after.json` shows the same cartridges in the same slots. +### Reboot persistence + +**Automated (soft reboot) — opt-in, two levels.** The +"Reboot persistence (AML soft reboot)" step of `hardware-destructive.yml` runs +`tests/hardware/test_reboot_persistence.py` only when: + +1. repo variable **`HARDWARE_REBOOT_TESTS`** = `true` (§2) — unset, the step is + skipped and a `::notice::` says so; and +2. every earlier step succeeded (`success()`, never `always()`): the test steps, + the junit gate, the recovery-report check and the lane's after-snapshot. + +The step itself exports `OPENBLADE_REBOOT_TESTS=enabled`; the test skips without +it, so a plain `pytest tests/hardware -m real_hardware` never reboots anything. +It runs under the same `hardware-destructive` environment approval, after the +lane's after-snapshot (so the lane diff is not polluted) and before the upload, +with `timeout-minutes: 20`. It pins `OPENBLADE_APPLIANCE_INSECURE=false`: a rig +with a self-signed appliance cert fails this step until the cert is trusted. + +The test refuses to reboot (fails loudly) unless the library is quiescent — no +live drive leases, no drive loaded, nothing mounted. It then snapshots inventory, +drive state and live leases, sends the documented `POST /aml/system/reboot` +`{"delay":0,"force":false}` (a dropped connection counts as accepted), polls +every 2 s until a login fails (bounded: 5 min — proves the reboot happened) and +then until a login succeeds (bounded: 10 min), re-snapshots, and fails on any +inventory/drive diff or lease change. It does **not** call +`recover_after_restart` (that function has no report-only mode and would release +leases); the lane's "Recovery report must be clean" step is the recovery +assertion. Only `reboot()` bypasses the read-only AML guard, for that one path; +a no-hardware unit test in the same file pins that `request()` still refuses +writes. + +**Manual (hard power-cycle) — fallback.** The runner cannot cut power. Once per +quarter, or whenever a soft reboot is not representative (firmware update, +power event): after a green destructive run, power-cycle the i3, wait for it to +come Ready, dispatch `hardware-library-smoke.yml`, and confirm the snapshot diff +against the pre-reboot `after.json` shows the same cartridges in the same slots. ## 6. What "no rig" looks like — and the no-green-by-skipping rule diff --git a/openblade/api/aml_state.py b/openblade/api/aml_state.py index 0665544..2c9723b 100644 --- a/openblade/api/aml_state.py +++ b/openblade/api/aml_state.py @@ -192,21 +192,41 @@ def _update_latency_stats(stats: dict[str, Any], value_ms: int, *, count: int) - def _get_admin_password() -> str: - pw = os.environ.get("OPENBLADE_ADMIN_PASSWORD", _DEFAULT_ADMIN_PASSWORD) - if pw == _DEFAULT_ADMIN_PASSWORD: + from openblade.config_validation import ConfigurationError, password_problem + + raw = os.environ.get("OPENBLADE_ADMIN_PASSWORD") + pw = raw if raw is not None else _DEFAULT_ADMIN_PASSWORD + problem = password_problem(raw) + if problem: + # Same fail-closed pattern as service_auth: production refuses weak values. + if os.environ.get("OPENBLADE_ENV", "development").strip().lower() == "production": + raise ConfigurationError( + f"OPENBLADE_ENV=production but OPENBLADE_ADMIN_PASSWORD {problem}." + ) _aml_log.warning( - "Using default admin password 'password'. " - "Set OPENBLADE_ADMIN_PASSWORD env var before deploying to production." + "OPENBLADE_ADMIN_PASSWORD %s. " + "Set a strong OPENBLADE_ADMIN_PASSWORD env var before deploying to production.", + problem, ) return pw def _get_service_password() -> str: - pw = os.environ.get("OPENBLADE_SERVICE_PASSWORD", _DEFAULT_SERVICE_PASSWORD) - if pw == _DEFAULT_SERVICE_PASSWORD: + from openblade.config_validation import ConfigurationError, password_problem + + raw = os.environ.get("OPENBLADE_SERVICE_PASSWORD") + pw = raw if raw is not None else _DEFAULT_SERVICE_PASSWORD + problem = password_problem(raw) + if problem: + # Same fail-closed pattern as service_auth: production refuses weak values. + if os.environ.get("OPENBLADE_ENV", "development").strip().lower() == "production": + raise ConfigurationError( + f"OPENBLADE_ENV=production but OPENBLADE_SERVICE_PASSWORD {problem}." + ) _aml_log.warning( - "Using default service password. " - "Set OPENBLADE_SERVICE_PASSWORD env var before deploying to production." + "OPENBLADE_SERVICE_PASSWORD %s. " + "Set a strong OPENBLADE_SERVICE_PASSWORD env var before deploying to production.", + problem, ) return pw diff --git a/openblade/api/main.py b/openblade/api/main.py index b16ea43..70eaba2 100644 --- a/openblade/api/main.py +++ b/openblade/api/main.py @@ -286,7 +286,13 @@ async def apply_aml_emulator_latency( @app.on_event("startup") async def initialize_aml_state() -> None: from openblade.api.aml_state import ensure_initialized + from openblade.config_validation import enforce_startup_config, startup_banner + # Production: raises ConfigurationError on any open auth gate (fail-closed). + # Elsewhere: one loud banner listing every open gate. + banner = startup_banner(enforce_startup_config(os.environ)) + if banner: + logging.getLogger("openblade.security").warning("%s", banner) log_api_auth_status() # Parse the fault profile at boot so a typo fails loudly here, not as a 500 on # the first request; this also anchors the reboot window at process start. diff --git a/openblade/api/nas_config.py b/openblade/api/nas_config.py index 9d2bc7b..771065d 100644 --- a/openblade/api/nas_config.py +++ b/openblade/api/nas_config.py @@ -27,6 +27,7 @@ ScratchThresholds, book_plan_capacity, plan_with_pool_budget, + release_plan_capacity, server_budget_request, ) from openblade.nas.catalog_rebuild import CatalogRebuildPlanner @@ -44,6 +45,7 @@ register_archive_plan, run_ingest_job, start_ingest_job, + unregister_archive_plan, ) from openblade.nas.ltfs_manifest import TapeMetadataWriter from openblade.nas.manifest_validator import ManifestValidator @@ -1232,6 +1234,12 @@ async def start_ingest( raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Archive plan is not safe to enqueue" ) + if plan.pool != request.pool_id: + # The booking is made against plan.pool; starting into any other pool (or a + # pool-less preview into a pool) would write to an unbudgeted pool. + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, detail="pool_id does not match plan pool" + ) if plan.ingest_mode is IngestMode.CACHE_DRIVE: if request.cache_drive_id is None: raise HTTPException( @@ -1257,25 +1265,40 @@ async def start_ingest( except ReservationError as exc: # Curated domain message (pool + plan id only); the pool budget is exhausted. raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc - if auto_clean_drives: - cleaned_drives = _auto_clean_required_drives() - job = start_ingest_job( - plan=plan, - dataset_name=request.dataset_name, - pool_id=request.pool_id, - nas_service=service, - cache_drive_id=request.cache_drive_id, - ) - if cleaned_drives: - job.notes.append(f"Auto-cleaned drives before ingest: {', '.join(cleaned_drives)}") - background_tasks.add_task( - run_ingest_job, - job.job_id, - nas_service=service, - library=context.library, - ltfs=context.ltfs, - cache_drive_id=request.cache_drive_id, - ) + try: + if auto_clean_drives: + cleaned_drives = _auto_clean_required_drives() + job = start_ingest_job( + plan=plan, + dataset_name=request.dataset_name, + pool_id=plan.pool, + nas_service=service, + cache_drive_id=request.cache_drive_id, + ) + if cleaned_drives: + job.notes.append(f"Auto-cleaned drives before ingest: {', '.join(cleaned_drives)}") + background_tasks.add_task( + run_ingest_job, + job.job_id, + nas_service=service, + library=context.library, + ltfs=context.ltfs, + cache_drive_id=request.cache_drive_id, + ) + except Exception: + # Anything failing after the booking must not leave pool budget held for 1h + # or a startable plan behind; re-raise the original error unchanged. + # Each cleanup step is isolated so a failing cleanup can never replace + # the original error (which is what the caller must see). + try: + release_plan_capacity(service.repository, plan.plan_id) + except Exception: # noqa: BLE001 - best-effort rollback; original error re-raised below + logger.exception("rollback: release_plan_capacity failed for %s", plan.plan_id) + try: + unregister_archive_plan(plan.plan_id) + except Exception: # noqa: BLE001 - best-effort rollback; original error re-raised below + logger.exception("rollback: unregister_archive_plan failed for %s", plan.plan_id) + raise return StartIngestResponse(job_id=job.job_id, dataset_id=job.dataset_id, status="running") diff --git a/openblade/api/routes_test_runner.py b/openblade/api/routes_test_runner.py index 9440fab..b074077 100644 --- a/openblade/api/routes_test_runner.py +++ b/openblade/api/routes_test_runner.py @@ -16,6 +16,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any, Literal, cast +from urllib.parse import urlsplit import structlog from fastapi import APIRouter, HTTPException @@ -129,9 +130,39 @@ def _parse_json_report(run_id: str) -> dict[str, Any]: # --------------------------------------------------------------------------- +# The runner spawns pytest with request-supplied AML URL/credentials, so it is a +# dev-only surface: opt-in, never in production, and never pointed off-host (SSRF). +TEST_RUNNER_ENABLED_ENV_VAR = "OPENBLADE_ENABLE_TEST_RUNNER" +_ALLOWED_AML_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "host.docker.internal"}) + + +def test_runner_enabled() -> bool: + """True only when explicitly enabled AND not running in production.""" + if os.environ.get("OPENBLADE_ENV", "development").strip().lower() == "production": + return False + return os.environ.get(TEST_RUNNER_ENABLED_ENV_VAR, "false").strip().lower() == "true" + + +def aml_url_allowed(url: str) -> bool: + """Allow only http(s) URLs whose host is loopback or host.docker.internal.""" + try: + parts = urlsplit(url) + host = (parts.hostname or "").lower() + except ValueError: + return False + return parts.scheme in {"http", "https"} and host in _ALLOWED_AML_HOSTS + + @router.post("/run", response_model=TestRunResponse) async def start_test_run(req: TestRunRequest) -> TestRunResponse: """Start a test run. Returns run_id. Output streams via /stream/{run_id}.""" + if not test_runner_enabled(): + raise HTTPException(status_code=404, detail="Not Found") + if not aml_url_allowed(req.i3_aml_url): + raise HTTPException( + status_code=400, + detail="i3_aml_url must be http(s) on localhost, 127.0.0.1, ::1 or host.docker.internal", + ) if ( req.target == "real" and os.environ.get("I3_REAL_HARDWARE_ENABLED", "false").lower() != "true" diff --git a/openblade/api/routes_upload.py b/openblade/api/routes_upload.py index 27bd8e9..363ab57 100644 --- a/openblade/api/routes_upload.py +++ b/openblade/api/routes_upload.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import logging import os import re import shutil @@ -57,6 +58,7 @@ def _safe_resolve(base: Path, file_id: str) -> Path: ReservationError, book_plan_capacity, plan_with_pool_budget, + release_plan_capacity, server_budget_request, ) from openblade.nas.ingest import ( @@ -64,6 +66,7 @@ def _safe_resolve(base: Path, file_id: str) -> Path: register_archive_plan, run_ingest_job, start_ingest_job, + unregister_archive_plan, ) from openblade.nas.service import NasService from openblade.nas.types import ( @@ -608,45 +611,64 @@ async def push_staged_files_to_share( plan = _plan_share_push(repo, ArchivePlanRequest.model_validate(request_data)) plan.policy_name = policy.name plan.policy_type = policy.policy_type - plan = register_archive_plan(plan) dataset_name = request.dataset_name or ( f"share-push-{pool_id}-{datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')}" ) + # Book before registering so a refused booking (409) leaves no orphan plan. try: book_plan_capacity(repo, plan, get_context().library, now=datetime.now(timezone.utc)) except ReservationError as exc: # Curated domain message (pool + plan id only); the pool budget is exhausted. raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc - job = start_ingest_job( - plan=plan, - dataset_name=dataset_name, - pool_id=pool_id, - nas_service=service, - cache_drive_id=None, - ) - context = get_context() - background_tasks.add_task( - _run_share_push_ingest_job, - job_id=job.job_id, - nas_service=service, - library=context.library, - ltfs=context.ltfs, - workspace=workspace, - repo=repo, - pool_id=pool_id, - staged_file_ids=staged_file_ids, - ) + plan = register_archive_plan(plan) + try: + job = start_ingest_job( + plan=plan, + dataset_name=dataset_name, + pool_id=pool_id, + nas_service=service, + cache_drive_id=None, + ) + context = get_context() + background_tasks.add_task( + _run_share_push_ingest_job, + job_id=job.job_id, + nas_service=service, + library=context.library, + ltfs=context.ltfs, + workspace=workspace, + repo=repo, + pool_id=pool_id, + staged_file_ids=staged_file_ids, + ) - return PushToShareResponse( - job_id=job.job_id, - dataset_id=job.dataset_id, - plan_id=plan.plan_id, - pushed_files=len(staged_file_ids), - pool_id=pool_id, - share_path=share.path, - status="running", - ) + return PushToShareResponse( + job_id=job.job_id, + dataset_id=job.dataset_id, + plan_id=plan.plan_id, + pushed_files=len(staged_file_ids), + pool_id=pool_id, + share_path=share.path, + status="running", + ) + except Exception: + # Roll back the booking and the registration; re-raise the original error. + # Each cleanup step is isolated so a failing cleanup can never replace + # the original error (which is what the caller must see). + try: + release_plan_capacity(repo, plan.plan_id) + except Exception: # noqa: BLE001 - best-effort rollback; original error re-raised below + logging.getLogger(__name__).exception( + "rollback: release_plan_capacity failed for %s", plan.plan_id + ) + try: + unregister_archive_plan(plan.plan_id) + except Exception: # noqa: BLE001 - best-effort rollback; original error re-raised below + logging.getLogger(__name__).exception( + "rollback: unregister_archive_plan failed for %s", plan.plan_id + ) + raise except Exception: shutil.rmtree(workspace, ignore_errors=True) raise diff --git a/openblade/catalog/repository.py b/openblade/catalog/repository.py index 4e4008d..f3e09b6 100644 --- a/openblade/catalog/repository.py +++ b/openblade/catalog/repository.py @@ -14,7 +14,7 @@ from typing import Any, Protocol, cast, runtime_checkable from uuid import uuid4 -from sqlalchemy import func, select +from sqlalchemy import delete, func, select from sqlalchemy.exc import OperationalError from sqlalchemy.orm import Session, selectinload @@ -2181,11 +2181,23 @@ def get_safety_token(self, token_value: str) -> SafetyToken | None: expires_at=row.expires_at, ) - def delete_safety_token(self, token_value: str) -> None: - row = self.session.get(SafetyTokenRecord, token_value) - if row is not None: - self.session.delete(row) - self.session.commit() + def delete_safety_token(self, token_value: str) -> bool: + """Atomically consume a token; True only for the one caller that deleted it. + + Single-use guarantee for destructive confirms: the DELETE runs inside + ``BEGIN IMMEDIATE`` on its own session (same pattern as the drive + leases), so two concurrent confirms with one token cannot both see + rowcount 1. + """ + with self._lock, self._lease_session() as session: + session.connection().exec_driver_sql("BEGIN IMMEDIATE") + result = session.execute( + delete(SafetyTokenRecord).where(SafetyTokenRecord.token == token_value) + ) + session.commit() + # A stale identity-map copy in `self.session` is harmless: a later + # get_safety_token may still see it, but this DELETE then returns False. + return bool(getattr(result, "rowcount", 0) == 1) # ── Drive leases (docs/decisions/2026-10-09-persistent-drive-leases.md) ── # diff --git a/openblade/config.py b/openblade/config.py index bf135dd..ee42c4a 100644 --- a/openblade/config.py +++ b/openblade/config.py @@ -2,12 +2,15 @@ from __future__ import annotations +import logging import os from dataclasses import dataclass from enum import Enum from pathlib import Path -from openblade.domain.errors import DriveCorrelationError +from openblade.domain.errors import DriveCorrelationError, RealHardwareDisabledError + +_LOG = logging.getLogger(__name__) class BackendMode(str, Enum): @@ -164,14 +167,33 @@ def _load_iblade_compat_mode() -> IBladeCompatibilityMode: return IBladeCompatibilityMode.EXTENDED +def _api_token_configured() -> bool: + return bool( + os.environ.get("OPENBLADE_API_TOKEN", "").strip() + or os.environ.get("OPENBLADE_API_TOKEN_FILE", "").strip() + ) + + def load_config() -> OpenBladeConfig: - backend_str = os.environ.get("OPENBLADE_BACKEND", "mock").lower() + backend_str = os.environ.get("OPENBLADE_BACKEND", "mock").strip().lower() + if backend_str == "simulator": # Dockerfile spelling; same as mock + backend_str = BackendMode.MOCK.value try: backend = BackendMode(backend_str) except ValueError: + _LOG.warning( + "Unrecognised OPENBLADE_BACKEND=%r; using the simulator (mock) backend", + backend_str.replace("\r", "").replace("\n", ""), + ) backend = BackendMode.MOCK real_hw = os.environ.get("OPENBLADE_REAL_HARDWARE_ENABLED", "false").lower() == "true" + if backend == BackendMode.REAL and real_hw and not _api_token_configured(): + # Third signal: no unauthenticated native API in front of real robotics. + raise RealHardwareDisabledError( + "OPENBLADE_BACKEND=real with OPENBLADE_REAL_HARDWARE_ENABLED=true " + "also requires OPENBLADE_API_TOKEN (or OPENBLADE_API_TOKEN_FILE) to be set" + ) drive_devices = tuple( device.strip() for device in os.environ.get("OPENBLADE_DRIVE_DEVICES", "").split(",") diff --git a/openblade/config_validation.py b/openblade/config_validation.py index 1581fdc..aa7c9f8 100644 --- a/openblade/config_validation.py +++ b/openblade/config_validation.py @@ -13,6 +13,7 @@ from collections.abc import Mapping from dataclasses import dataclass +from urllib.parse import urlsplit BLOCKING = "blocking" WARNING = "warning" @@ -22,6 +23,28 @@ # Kept in sync with openblade/api/aml_state.py::_DEFAULT_ADMIN_PASSWORD / _DEFAULT_SERVICE_PASSWORD DEFAULT_ADMIN_PASSWORD = "password" DEFAULT_SERVICE_PASSWORD = "service123" +MIN_PASSWORD_LENGTH = 12 +_KNOWN_DEFAULT_PASSWORDS = frozenset({DEFAULT_ADMIN_PASSWORD, DEFAULT_SERVICE_PASSWORD}) + + +def password_problem(value: str | None) -> str | None: + """Why ``value`` is not an acceptable AML password, or None if it is. + + Surrounding whitespace is ignored for the check, so ``" "`` counts as unset + (it would otherwise satisfy compose's ``${VAR:?}``). Shared with + ``openblade/api/aml_state.py`` so the startup gate and the runtime getters + cannot disagree. Never includes the value itself. + """ + pw = (value or "").strip() + if not pw: + return "is unset or blank" + if pw in _KNOWN_DEFAULT_PASSWORDS: + return "is a known insecure default" + if len(pw) < MIN_PASSWORD_LENGTH: + return f"is shorter than {MIN_PASSWORD_LENGTH} characters" + return None + + VALID_BACKENDS = {"mock", "simulator", "real"} @@ -32,6 +55,26 @@ class Finding: message: str +def redact_url(url: str) -> str: + """Return ``url`` as scheme://host[:port]/path with credentials as ``***``. + + Query and fragment are dropped (they can carry ``password=`` parameters). + Every finding message that quotes a URL goes through this, because + WARNING findings are logged verbatim by the startup banner. + """ + try: + parts = urlsplit(url) + host = parts.hostname or "" + port = parts.port + except ValueError: + return "" + if port is not None: + host = f"{host}:{port}" + creds = "***@" if (parts.username is not None or parts.password is not None) else "" + scheme = f"{parts.scheme}://" if parts.scheme else "" + return f"{scheme}{creds}{host}{parts.path}" + + def validate_config(env: Mapping[str, str]) -> list[Finding]: """Return findings for the given environment. Deploy-blocking = any BLOCKING.""" findings: list[Finding] = [] @@ -59,6 +102,20 @@ def validate_config(env: Mapping[str, str]) -> list[Finding]: ) ) + if ( + backend == "real" + and env.get("OPENBLADE_REAL_HARDWARE_ENABLED", "false").strip().lower() == "true" + and not env.get("OPENBLADE_API_TOKEN", "").strip() + and not env.get("OPENBLADE_API_TOKEN_FILE", "").strip() + ): + findings.append( + Finding( + BLOCKING, + "real_backend_requires_api_token", + "OPENBLADE_BACKEND=real with OPENBLADE_REAL_HARDWARE_ENABLED=true requires OPENBLADE_API_TOKEN (or OPENBLADE_API_TOKEN_FILE)", + ) + ) + if env.get( "OPENBLADE_ROBOTICS_TRANSPORT", "scsi" ).strip().lower() == "webservices" and not env.get("OPENBLADE_SCALAR_URL"): @@ -70,27 +127,94 @@ def validate_config(env: Mapping[str, str]) -> list[Finding]: ) ) + # --- open gates: blocking in production, one loud banner line otherwise --- + sev = BLOCKING if is_prod else WARNING + if ( + not env.get("OPENBLADE_API_TOKEN", "").strip() + and not env.get("OPENBLADE_API_TOKEN_FILE", "").strip() + ): + findings.append( + Finding( + sev, + "native_api_open", + "NATIVE API OPEN: OPENBLADE_API_TOKEN / OPENBLADE_API_TOKEN_FILE unset; " + "every native mutating route is unauthenticated", + ) + ) + origins = [o.strip() for o in env.get("OPENBLADE_CORS_ORIGINS", "").split(",")] + if "*" in origins: + findings.append( + Finding( + sev, + "cors_wildcard", + "CORS WILDCARD: OPENBLADE_CORS_ORIGINS contains '*' (credentials are allowed)", + ) + ) + if not is_prod: + problem = password_problem(env.get("OPENBLADE_ADMIN_PASSWORD")) + if problem: + findings.append( + Finding( + WARNING, + "default_admin_password", + f"WEAK AML ADMIN PASSWORD: OPENBLADE_ADMIN_PASSWORD {problem}", + ) + ) + problem = password_problem(env.get("OPENBLADE_SERVICE_PASSWORD")) + if problem: + findings.append( + Finding( + WARNING, + "default_service_password", + f"WEAK AML SERVICE PASSWORD: OPENBLADE_SERVICE_PASSWORD {problem}", + ) + ) + if env.get("OPENBLADE_SERVICE_TOKEN", DEFAULT_SERVICE_TOKEN) == DEFAULT_SERVICE_TOKEN: + findings.append( + Finding( + WARNING, + "default_service_token", + "DEFAULT SERVICE TOKEN: OPENBLADE_SERVICE_TOKEN is unset or the default", + ) + ) + + # Mirrors web_flask/app.py::_bool_env: any set, non-truthy value is "off". + cookies = env.get("OPENBLADE_WEB_SECURE_COOKIES") + if ( + is_prod + and cookies is not None + and cookies.strip().lower() not in {"1", "true", "yes", "on"} + ): + findings.append( + Finding( + WARNING, + "insecure_session_cookies", + "INSECURE FLASK COOKIES: OPENBLADE_WEB_SECURE_COOKIES=false in production " + "(session cookies cross the LAN in cleartext; put TLS in front)", + ) + ) + if not is_prod: return findings # --- production-only blocking checks --- - admin = env.get("OPENBLADE_ADMIN_PASSWORD", "") - if not admin or admin == DEFAULT_ADMIN_PASSWORD: + problem = password_problem(env.get("OPENBLADE_ADMIN_PASSWORD")) + if problem: findings.append( Finding( BLOCKING, "unsafe_admin_password", - "OPENBLADE_ADMIN_PASSWORD is unset or the insecure default in production", + f"OPENBLADE_ADMIN_PASSWORD {problem} (production)", ) ) - svc_pw = env.get("OPENBLADE_SERVICE_PASSWORD", "") - if not svc_pw or svc_pw == DEFAULT_SERVICE_PASSWORD: + problem = password_problem(env.get("OPENBLADE_SERVICE_PASSWORD")) + if problem: findings.append( Finding( BLOCKING, "missing_service_password", - "OPENBLADE_SERVICE_PASSWORD is unset or the insecure default in production", + f"OPENBLADE_SERVICE_PASSWORD {problem} (production)", ) ) @@ -114,7 +238,7 @@ def validate_config(env: Mapping[str, str]) -> list[Finding]: Finding( WARNING, "dev_db_path", - f"OPENBLADE_DB_URL={db_url!r} looks like a development/default path", + f"OPENBLADE_DB_URL={redact_url(db_url)!r} looks like a development/default path", ) ) @@ -143,3 +267,50 @@ def validate_config(env: Mapping[str, str]) -> list[Finding]: def is_deployable(findings: list[Finding]) -> bool: return not any(f.severity == BLOCKING for f in findings) + + +# Codes the API process refuses to start on in production (startup gate). Other +# BLOCKING findings (e.g. missing_db_url) remain deploy-gate-only. +STARTUP_REFUSAL_CODES = frozenset( + { + "native_api_open", + "cors_wildcard", + "unsafe_admin_password", + "missing_service_password", + "unsafe_service_token", + } +) + + +class ConfigurationError(RuntimeError): + """Raised at startup when production configuration leaves an auth gate open. + + The message names variables/codes only; it never echoes configured values. + """ + + +def startup_banner(findings: list[Finding]) -> str: + """One multi-line banner listing every open gate (empty string if none).""" + if not findings: + return "" + bar = "!" * 72 + lines = [bar, "OPENBLADE SECURITY: the following gates are OPEN"] + lines += [f" - {f.message}" for f in findings] + lines.append(bar) + return "\n".join(lines) + + +def enforce_startup_config(env: Mapping[str, str]) -> list[Finding]: + """Refuse to start in production with an open auth gate; else return findings. + + Collects every problem and reports them together. Outside production the + caller logs :func:`startup_banner` of the returned findings. + """ + findings = validate_config(env) + refusals = [f for f in findings if f.severity == BLOCKING and f.code in STARTUP_REFUSAL_CODES] + if refusals: + raise ConfigurationError( + "Refusing to start in OPENBLADE_ENV=production: " + + "; ".join(f.message for f in refusals) + ) + return findings diff --git a/openblade/domain/media_allowlist.py b/openblade/domain/media_allowlist.py new file mode 100644 index 0000000..37f007c --- /dev/null +++ b/openblade/domain/media_allowlist.py @@ -0,0 +1,66 @@ +"""Scratch/sacrificial media allowlist — the single parser for barcode lists. + +Used by the CI hardware lanes (``tools/hardware/allowlist.py``) and, in code, by +``FormatService`` on a real backend: only barcodes in ``OPENBLADE_SCRATCH_BARCODES`` +may be formatted, and an empty allowlist refuses everything (fail closed). +""" + +from __future__ import annotations + +import os +import re +from collections.abc import Sequence + +SCRATCH_BARCODES_ENV = "OPENBLADE_SCRATCH_BARCODES" + +_SEPARATORS = re.compile(r"[\s,]+") + + +class AllowlistError(Exception): + """The requested barcodes are not safe to use destructively.""" + + +def parse_barcodes(raw: str | None) -> list[str]: + """Split a comma/whitespace list into upper-cased, de-duplicated barcodes.""" + seen: dict[str, None] = {} + for token in _SEPARATORS.split(raw or ""): + if token: + seen.setdefault(token.strip().upper(), None) + return list(seen) + + +def format_barcodes(barcodes: Sequence[str]) -> str: + """The normalized stdout form: comma-joined, no spaces.""" + return ",".join(barcodes) + + +def check_subset(requested: Sequence[str], allowlist: Sequence[str]) -> list[str]: + """Return ``requested`` if it is a non-empty subset of a non-empty allowlist. + + Raises :class:`AllowlistError` naming the offenders otherwise. + """ + if not allowlist: + raise AllowlistError( + "SACRIFICIAL_BARCODES is empty: no tape is approved for destructive use (fail closed)." + ) + if not requested: + raise AllowlistError("No barcodes requested (pass --barcodes or set BARCODES).") + allowed = set(allowlist) + offenders = [b for b in requested if b not in allowed] + if offenders: + raise AllowlistError( + "Barcodes outside the sacrificial allowlist: " + + ", ".join(offenders) + + ". Refusing before touching hardware." + ) + return list(requested) + + +def scratch_barcodes_from_env() -> list[str]: + """The operator's scratch allowlist from ``OPENBLADE_SCRATCH_BARCODES``.""" + return parse_barcodes(os.environ.get(SCRATCH_BARCODES_ENV)) + + +def is_scratch_barcode(barcode: str, allowlist: Sequence[str]) -> bool: + """True only if ``barcode`` is in a NON-empty allowlist (fail closed).""" + return bool(allowlist) and barcode.strip().upper() in set(allowlist) diff --git a/openblade/hardware/appliance_client.py b/openblade/hardware/appliance_client.py new file mode 100644 index 0000000..180f098 --- /dev/null +++ b/openblade/hardware/appliance_client.py @@ -0,0 +1,97 @@ +"""Read-only cookie-session client for the real appliance's AML HTTP API. + +Shared by the read-only hardware lane (``tests/hardware/readonly``) and the +day-one preflight tool (``tools/hardware/preflight.py``). Every request passes +through :func:`check_appliance_request`, which refuses anything but a GET or +the login/logout handshake, so neither caller can mutate the rig. +""" + +from __future__ import annotations + +import json +import ssl +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from http.cookiejar import CookieJar +from typing import Any + + +class ReadOnlyViolation(RuntimeError): + """A read-only-lane test attempted an HTTP request or command that could mutate the rig.""" + + +# The only non-GET requests the read-only lane may send: the session handshake. +AUTH_REQUESTS = frozenset({("POST", "/aml/users/login"), ("POST", "/aml/auth/logout")}) + + +def check_appliance_request(method: str, path: str) -> None: + """Raise ReadOnlyViolation unless ``method path`` is a GET or the login/logout call. + + The method is matched exactly (HTTP methods are case-sensitive), so ``"get"`` + is refused rather than normalised. + """ + verb = method + if not path.startswith("/"): + raise ReadOnlyViolation(f"appliance path must be absolute, got {path!r}") + if verb == "GET" or (verb, path) in AUTH_REQUESTS: + return + raise ReadOnlyViolation(f"read-only lane refuses {verb} {path}") + + +@dataclass +class ApplianceResponse: + status: int + body: Any + + +@dataclass +class ApplianceClient: + """Minimal cookie-session HTTP client for the real appliance's AML API.""" + + base_url: str + user: str + password: str + insecure: bool = False + timeout_s: float = 30.0 + _opener: urllib.request.OpenerDirector = field(init=False) + + def __post_init__(self) -> None: + context = ssl.create_default_context() + if self.insecure: + # Opt-in via OPENBLADE_APPLIANCE_INSECURE=true for a rig whose + # appliance only has its factory self-signed certificate. + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + self._opener = urllib.request.build_opener( + urllib.request.HTTPCookieProcessor(CookieJar()), + urllib.request.HTTPSHandler(context=context), + ) + + def request(self, method: str, path: str, json_body: Any = None) -> ApplianceResponse: + check_appliance_request(method, path) + data = None if json_body is None else json.dumps(json_body).encode() + req = urllib.request.Request( # noqa: S310 - base URL is operator-set rig config; both callers check the scheme before constructing the client + self.base_url.rstrip("/") + path, + data=data, + method=method, + headers={"Content-Type": "application/json", "Accept": "application/json"}, + ) + try: + with self._opener.open(req, timeout=self.timeout_s) as resp: + status, raw = resp.status, resp.read() + except urllib.error.HTTPError as exc: + status, raw = exc.code, exc.read() + try: + body: Any = json.loads(raw) if raw else None + except ValueError: + body = None + return ApplianceResponse(status=status, body=body) + + def login(self) -> ApplianceResponse: + return self.request( + "POST", "/aml/users/login", {"name": self.user, "password": self.password} + ) + + def logout(self) -> ApplianceResponse: + return self.request("POST", "/aml/auth/logout") diff --git a/openblade/hardware/safety.py b/openblade/hardware/safety.py index cebb4a9..35032ec 100644 --- a/openblade/hardware/safety.py +++ b/openblade/hardware/safety.py @@ -12,6 +12,10 @@ def require_real_hardware(config: OpenBladeConfig) -> RealHardwareGuard: "Real hardware operations require OPENBLADE_BACKEND=real " "and OPENBLADE_REAL_HARDWARE_ENABLED=true" ) + # NOTE: "config-verified" is NOT a human acknowledgment; it only satisfies + # the guard's non-empty check. The real gates are the config flags above, + # the OPENBLADE_API_TOKEN requirement in load_config, and the format + # token + scratch allowlist. Removing the field ripples into many callers. return RealHardwareGuard( config_backend=config.backend.value, config_real_hardware_enabled=config.real_hardware_enabled, diff --git a/openblade/jobs/format.py b/openblade/jobs/format.py index 604d582..04ce0c2 100644 --- a/openblade/jobs/format.py +++ b/openblade/jobs/format.py @@ -2,27 +2,72 @@ from __future__ import annotations +from collections.abc import Sequence + from openblade.catalog.repository import CatalogRepository +from openblade.config import BackendMode, load_config from openblade.domain.backends import LibraryBackend, LTFSBackend from openblade.domain.errors import BarcodeMismatchError, FormatRequiresConfirmationError +from openblade.domain.media_allowlist import ( + SCRATCH_BARCODES_ENV, + is_scratch_barcode, + scratch_barcodes_from_env, +) from openblade.domain.models import OperationResult from openblade.domain.policies import DryRunPlan, FormatConfirmation, SafetyToken from openblade.nas.tape_orchestrator import execute_tape_request from openblade.nas.types import TapeOpRequest, TapeOpType +def normalize_barcode(barcode: str) -> str: + """Service-boundary form (``strip().upper()``), matching the scratch allowlist. + + Applied once on entry so allowlist, inventory lookup, token and catalog all + see the same barcode (``" abc123"`` used to pass the allowlist, then fail as + unknown). + """ + return barcode.strip().upper() + + +def real_backend_from_config() -> bool: + """True when the configured backend is REAL, ignoring the enable flag (fail closed). + + The single detection used by ``FormatService`` and by NAS ingest's refusal to + auto-format on real hardware. + """ + return load_config().backend == BackendMode.REAL + + class FormatService: def __init__( self, catalog: CatalogRepository, library: LibraryBackend, ltfs: LTFSBackend, + *, + real_backend: bool | None = None, + scratch_barcodes: Sequence[str] | None = None, ) -> None: self.catalog = catalog self.library = library self.ltfs = ltfs + # Real media gate: on a REAL backend only allowlisted scratch tapes may be + # formatted. Defaults come from the environment so every construction + # site is covered; the backend check ignores the enable flag (fail closed). + self._real_backend = real_backend_from_config() if real_backend is None else real_backend + self._scratch_barcodes = ( + scratch_barcodes_from_env() if scratch_barcodes is None else list(scratch_barcodes) + ) + + def _require_scratch_media(self, barcode: str) -> None: + if self._real_backend and not is_scratch_barcode(barcode, self._scratch_barcodes): + raise FormatRequiresConfirmationError( + f"Barcode {barcode} is not in {SCRATCH_BARCODES_ENV}; real-hardware format refused" + ) def dry_run(self, barcode: str) -> tuple[DryRunPlan, SafetyToken]: + barcode = normalize_barcode(barcode) + self._require_scratch_media(barcode) inventory = self.library.inventory() known = {str(slot.barcode) for slot in inventory.slots if slot.barcode is not None} | { str(drive.barcode) for drive in inventory.drives if drive.barcode is not None @@ -42,11 +87,17 @@ def dry_run(self, barcode: str) -> tuple[DryRunPlan, SafetyToken]: return plan, token def confirm(self, barcode: str, token_value: str) -> OperationResult: + barcode = normalize_barcode(barcode) + self._require_scratch_media(barcode) token = self.catalog.get_safety_token(token_value) if token is None: raise FormatRequiresConfirmationError("Unknown or missing safety token") if token.operation != "format" or token.target_barcode != barcode: raise FormatRequiresConfirmationError("Safety token does not authorize this barcode") + # Consume BEFORE the job: single-use even under concurrent confirms, and + # a failed job does not re-arm the token (operator re-runs dry-run). + if not self.catalog.delete_safety_token(token_value): + raise FormatRequiresConfirmationError("Safety token already used or expired") result = run_format_job( barcode, FormatConfirmation(expected_barcode=barcode, safety_token=token), @@ -54,7 +105,6 @@ def confirm(self, barcode: str, token_value: str) -> OperationResult: self.ltfs, self.catalog, ) - self.catalog.delete_safety_token(token_value) cartridge = self.catalog.add_cartridge(barcode) cartridge.formatted = True cartridge.used_bytes = 0 diff --git a/openblade/nas/capacity.py b/openblade/nas/capacity.py index 4580f55..067cc74 100644 --- a/openblade/nas/capacity.py +++ b/openblade/nas/capacity.py @@ -238,6 +238,18 @@ def book_plan_capacity( raise ReservationError(f"pool {pool_id} cannot reserve capacity for plan {plan.plan_id}") +def release_plan_capacity(repository: CatalogRepository, plan_id: str) -> None: + """Roll back a start-path booking; never raises, so the caller's error surfaces. + + Used when a start fails after ``book_plan_capacity``. A failed release is logged + and left to expire by TTL rather than masking the original exception. + """ + try: + repository.delete_nas_reservations_for_owner(plan_id) + except Exception as exc: # noqa: BLE001 - a failed release must not replace the start failure being re-raised; the booking expires by TTL + logger.warning("reservation rollback failed for plan %s", plan_id, exc_info=exc) + + class CapacityReservation(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/openblade/nas/ingest.py b/openblade/nas/ingest.py index 6227de9..0354ba2 100644 --- a/openblade/nas/ingest.py +++ b/openblade/nas/ingest.py @@ -14,9 +14,14 @@ from openblade.catalog.repository import CatalogRepository from openblade.domain.backends import LibraryBackend, LTFSBackend -from openblade.domain.errors import OpenBladeError, safe_job_error +from openblade.domain.errors import ( + FormatRequiresConfirmationError, + OpenBladeError, + safe_job_error, +) from openblade.domain.models import MountHandle, MountMode from openblade.domain.policies import FormatConfirmation, SafetyToken +from openblade.jobs.format import real_backend_from_config from openblade.nas.archive_lifecycle import ( ArchiveLifecycleManager, ArchiveLifecycleResult, @@ -144,6 +149,7 @@ def __init__( service: NasService, library: LibraryBackend, ltfs: LTFSBackend, + real_backend: bool | None = None, ) -> None: self.job = job self.dataset = dataset @@ -153,6 +159,8 @@ def __init__( self._load_tape = library.load self._unload_tape = library.unload self._format_tape = ltfs.format + # Same detection FormatService uses: on a real backend ingest never formats. + self._real_backend = real_backend_from_config() if real_backend is None else real_backend self._write_tape_bytes = ltfs.write_bytes self._file_record_ids = { record.relative_path: record.id for record in service.list_file_records(dataset.id) @@ -340,6 +348,15 @@ def _ensure_formatted(self, barcode: str) -> None: tape = self.ltfs.ensure_tape(barcode) if tape.formatted: return + if self._real_backend: + # Never mkltfs from ingest on real hardware: formatting is only the + # two-phase /ltfs/format flow (scratch allowlist + persisted single-use + # token). Typed, so run() marks the job FAILED with this curated text. + raise FormatRequiresConfirmationError( + f"Tape {barcode} is not LTFS-formatted; ingest will not format on real " + "hardware - format it via the two-phase /ltfs/format flow" + ) + # Simulator only: direct auto-format with a self-minted token. self._format_tape( barcode, FormatConfirmation( @@ -698,6 +715,12 @@ def register_archive_plan(plan: ArchivePlan) -> ArchivePlan: return plan +def unregister_archive_plan(plan_id: str) -> None: + """Drop a plan from the in-memory registry (no-op if absent).""" + with _STORE_LOCK: + ArchivePlanStore.pop(plan_id, None) + + def get_archive_plan(plan_id: str) -> ArchivePlan | None: with _STORE_LOCK: return ArchivePlanStore.get(plan_id) diff --git a/scripts/mhvtl/env.sh b/scripts/mhvtl/env.sh index 4ff06cb..dfd5b08 100755 --- a/scripts/mhvtl/env.sh +++ b/scripts/mhvtl/env.sh @@ -46,8 +46,14 @@ EOF [ -n "$drive_nst" ] || rig_die "no tape drives found on this rig" +# Real mode refuses to start without an API token (openblade/config.py raises +# RealHardwareDisabledError). Keep the caller's (the workflow mints and masks +# one per run); otherwise mint a throwaway one for a local run. +OPENBLADE_API_TOKEN="${OPENBLADE_API_TOKEN:-$(openssl rand -hex 32)}" + cat < list[dict[str, Any]]: return [json.loads(p.read_text()) for p in sorted(CORPUS_DIR.rglob("*.json"))] -class ReadOnlyViolation(RuntimeError): - """A read-only-lane test attempted an HTTP request or command that could mutate the rig.""" - - -# The only non-GET requests the read-only lane may send: the session handshake. -AUTH_REQUESTS = frozenset({("POST", "/aml/users/login"), ("POST", "/aml/auth/logout")}) - - -def check_appliance_request(method: str, path: str) -> None: - """Raise ReadOnlyViolation unless ``method path`` is a GET or the login/logout call. - - The method is matched exactly (HTTP methods are case-sensitive), so ``"get"`` - is refused rather than normalised. - """ - verb = method - if not path.startswith("/"): - raise ReadOnlyViolation(f"appliance path must be absolute, got {path!r}") - if verb == "GET" or (verb, path) in AUTH_REQUESTS: - return - raise ReadOnlyViolation(f"read-only lane refuses {verb} {path}") - - _DEVICE = re.compile(r"/dev/[A-Za-z0-9_./:+-]+") _LOG_PAGE = re.compile(r"(0x)?[0-9A-Fa-f]{1,2}") @@ -100,63 +83,6 @@ def run( return super().run(args, timeout=timeout, redact_args=redact_args) -@dataclass -class ApplianceResponse: - status: int - body: Any - - -@dataclass -class ApplianceClient: - """Minimal cookie-session HTTP client for the real appliance's AML API.""" - - base_url: str - user: str - password: str - insecure: bool = False - _opener: urllib.request.OpenerDirector = field(init=False) - - def __post_init__(self) -> None: - context = ssl.create_default_context() - if self.insecure: - # Opt-in via OPENBLADE_APPLIANCE_INSECURE=true for a rig whose - # appliance only has its factory self-signed certificate. - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - self._opener = urllib.request.build_opener( - urllib.request.HTTPCookieProcessor(CookieJar()), - urllib.request.HTTPSHandler(context=context), - ) - - def request(self, method: str, path: str, json_body: Any = None) -> ApplianceResponse: - check_appliance_request(method, path) - data = None if json_body is None else json.dumps(json_body).encode() - req = urllib.request.Request( # noqa: S310 - base URL is operator-set rig config, scheme checked in fixture - self.base_url.rstrip("/") + path, - data=data, - method=method, - headers={"Content-Type": "application/json", "Accept": "application/json"}, - ) - try: - with self._opener.open(req, timeout=30) as resp: - status, raw = resp.status, resp.read() - except urllib.error.HTTPError as exc: - status, raw = exc.code, exc.read() - try: - body: Any = json.loads(raw) if raw else None - except ValueError: - body = None - return ApplianceResponse(status=status, body=body) - - def login(self) -> ApplianceResponse: - return self.request( - "POST", "/aml/users/login", {"name": self.user, "password": self.password} - ) - - def logout(self) -> ApplianceResponse: - return self.request("POST", "/aml/auth/logout") - - @pytest.fixture def appliance(real_hardware_guard) -> ApplianceClient: url = os.environ.get("OPENBLADE_APPLIANCE_URL", "") diff --git a/tests/hardware/test_reboot_persistence.py b/tests/hardware/test_reboot_persistence.py new file mode 100644 index 0000000..7a0635b --- /dev/null +++ b/tests/hardware/test_reboot_persistence.py @@ -0,0 +1,164 @@ +"""Reboot persistence on the real appliance (destructive lane, environment-approved). + +Reboots the appliance through the documented AML endpoint ``POST /aml/system/reboot``, +waits for its AML API to go down and come back, and asserts the library inventory, +drive state and OpenBlade's live drive leases are identical across the reboot. + +Precondition (fails loudly otherwise): the library is quiescent — no live leases, +no drive loaded, nothing mounted. A reboot with a mounted drive is disruptive, and +"leases identical" would be vacuous with leases in flight. + +The test does NOT call ``recover_after_restart`` itself: it has no report-only mode +(it fails jobs and releases leases), so the lane's existing "Recovery report must be +clean" step is the recovery assertion. + +A reboot does not touch media, but it interrupts every client of the appliance, so +the test additionally requires ``OPENBLADE_REBOOT_TESTS=enabled`` (mirroring +``OPENBLADE_FAULT_TESTS``): ``pytest tests/hardware -m real_hardware`` alone never +reboots anything. A hard power-cycle stays a manual step — see +docs/runbooks/hardware-lanes.md "Reboot persistence". +""" + +from __future__ import annotations + +import http.client +import json +import os +import time +import urllib.error +import urllib.request +from typing import Any + +import pytest + +from openblade.bootstrap import create_context +from openblade.config import load_config +from tests.hardware.readonly.conftest import ( + ApplianceClient, + ApplianceResponse, + ReadOnlyViolation, +) +from tools.hardware.snapshot import capture_snapshot, diff_snapshots + +REBOOT_PATH = "/aml/system/reboot" +# Bounded waits (the workflow step's timeout-minutes covers their sum). The +# appliance must stop answering within DOWN_TIMEOUT_S of accepting the reboot... +DOWN_TIMEOUT_S = 300.0 +# ...and accept a login again within UP_TIMEOUT_S after it went down. +UP_TIMEOUT_S = 600.0 +# Short enough that a brief downtime window is not missed between polls. +POLL_INTERVAL_S = 2.0 +# Errors after the POST that mean "connection dropped mid-response", i.e. the +# appliance accepted the reboot and went down before answering. +_DROPPED = (urllib.error.URLError, OSError, http.client.HTTPException) + + +class RebootClient(ApplianceClient): + """ApplianceClient plus the single non-read-only call this test needs. + + ``request`` keeps the read-only guard unchanged; only ``reboot`` bypasses + it, and only for exactly ``POST /aml/system/reboot`` (hard-coded below). + """ + + def reboot(self) -> ApplianceResponse: + """POST the reboot; a dropped connection counts as accepted (status 200).""" + req = urllib.request.Request( # noqa: S310 - base URL is operator-set rig config, scheme checked in fixture + self.base_url.rstrip("/") + REBOOT_PATH, + data=json.dumps({"delay": 0, "force": False}).encode(), + method="POST", + headers={"Content-Type": "application/json", "Accept": "application/json"}, + ) + try: + with self._opener.open(req, timeout=30) as resp: + return ApplianceResponse(status=resp.status, body=None) + except urllib.error.HTTPError as exc: + return ApplianceResponse(status=exc.code, body=None) + except _DROPPED: + return ApplianceResponse(status=200, body=None) + + def login_ok(self) -> bool: + """True iff a login succeeds now; an unreachable appliance is False.""" + try: + return self.login().status == 200 + except _DROPPED: + return False + + +@pytest.fixture +def reboot_tests_enabled() -> None: + if os.environ.get("OPENBLADE_REBOOT_TESTS") != "enabled": + pytest.skip("OPENBLADE_REBOOT_TESTS must be set to 'enabled'") + + +@pytest.fixture +def reboot_client(real_hardware_guard, reboot_tests_enabled) -> RebootClient: + del real_hardware_guard, reboot_tests_enabled + url = os.environ.get("OPENBLADE_APPLIANCE_URL", "") + user = os.environ.get("OPENBLADE_APPLIANCE_USER", "") + password = os.environ.get("OPENBLADE_APPLIANCE_PASSWORD", "") + if not (url and user and password): + pytest.skip( + "Appliance AML API not configured: set OPENBLADE_APPLIANCE_URL, " + "OPENBLADE_APPLIANCE_USER and OPENBLADE_APPLIANCE_PASSWORD" + ) + if not url.startswith(("https://", "http://")): + pytest.fail(f"OPENBLADE_APPLIANCE_URL must be http(s)://, got scheme of {url[:8]!r}") + insecure = os.environ.get("OPENBLADE_APPLIANCE_INSECURE") == "true" + return RebootClient(base_url=url, user=user, password=password, insecure=insecure) + + +def _wait_for(predicate: Any, timeout_s: float, what: str) -> None: + deadline = time.monotonic() + timeout_s + while not predicate(): + if time.monotonic() >= deadline: + pytest.fail(f"appliance {what} within {timeout_s:.0f}s") + time.sleep(POLL_INTERVAL_S) + + +def _state() -> dict[str, Any]: + context = create_context(load_config()) + leases = sorted( + (lease.id, lease.drive_id, lease.job_id, lease.barcode, lease.physical_drive_id) + for lease in context.catalog.live_leases() + ) + return {"snapshot": capture_snapshot(context.library), "leases": leases} + + +def _assert_quiescent(state: dict[str, Any]) -> None: + assert state["leases"] == [], f"refusing to reboot with live leases: {state['leases']}" + busy = [ + d + for d in state["snapshot"]["drives"] + if d["drive_state"] != "empty" or d["mount_state"] != "unmounted" or d["barcode"] + ] + assert not busy, f"refusing to reboot with a loaded or mounted drive: {busy}" + + +def test_reboot_client_request_keeps_read_only_guard() -> None: + """No hardware: the bypass is ``reboot()`` only; ``request`` refuses writes.""" + client = RebootClient(base_url="https://appliance.invalid", user="u", password="p") + with pytest.raises(ReadOnlyViolation): + client.request("POST", "/aml/media/move") + with pytest.raises(ReadOnlyViolation): + client.request("POST", REBOOT_PATH) + + +@pytest.mark.real_hardware +def test_reboot_preserves_inventory_and_leases(reboot_client: RebootClient) -> None: + before = _state() + _assert_quiescent(before) + + assert reboot_client.login().status == 200, "appliance login failed before reboot" + resp = reboot_client.reboot() + assert 200 <= resp.status < 300, f"POST {REBOOT_PATH} rejected: HTTP {resp.status}" + + # Seeing the API go down first is what makes the later login prove a reboot + # happened, rather than racing the scheduled reboot and passing vacuously. + _wait_for(lambda: not reboot_client.login_ok(), DOWN_TIMEOUT_S, "never went down") + _wait_for(reboot_client.login_ok, UP_TIMEOUT_S, "did not accept a login again") + reboot_client.logout() + + after = _state() + diff = diff_snapshots(before["snapshot"], after["snapshot"]) + assert not diff["changed"], f"inventory/drive state changed across reboot: {diff}" + assert after["leases"] == before["leases"], "live drive leases changed across reboot" diff --git a/tests/unit/test_capacity_share_push_safety.py b/tests/unit/test_capacity_share_push_safety.py new file mode 100644 index 0000000..f59ae34 --- /dev/null +++ b/tests/unit/test_capacity_share_push_safety.py @@ -0,0 +1,113 @@ +"""Share-push start: booking precedes plan registration; failures leave no orphans.""" + +from __future__ import annotations + +import io +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from httpx import Response + +from openblade.api.main import app +from openblade.bootstrap import create_context, get_context, reset_context +from openblade.config import OpenBladeConfig +from openblade.nas.capacity import ReservationError +from openblade.nas.ingest import ArchivePlanStore +from openblade.nas.service import NasService +from openblade.nas.types import NasPool, NasShareDefinition + + +class _Boom(RuntimeError): + pass + + +@pytest.fixture() +def client(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> TestClient: + monkeypatch.setenv("OPENBLADE_STAGING_DIR", str(tmp_path / "staging")) + monkeypatch.setenv("OPENBLADE_RESTORE_DIR", str(tmp_path / "restore")) + reset_context(create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 'p.db'}"))) + service = NasService(get_context().catalog) + service.upsert_pool(NasPool(id="1", name="Pool 1", default_policy_id="balanced")) + service.upsert_share( + NasShareDefinition( + path="/shares/pool-1", + name="Pool 1 Share", + share_type="pool", + pool_ids=["1"], + default_policy_id="balanced", + ) + ) + return TestClient(app) + + +def _push(client: TestClient) -> Response: + login = client.post("/aml/users/login", json={"name": "admin", "password": "password"}) + headers = {"Cookie": f"sessionID={login.cookies.get('sessionID')}"} + upload = client.post( + "/api/pools/1/upload", + files={"file": ("push.txt", io.BytesIO(b"payload"), "text/plain")}, + headers=headers, + ) + assert upload.status_code == 200 + return client.post( + "/api/pools/1/push-to-share", + json={"share_path": "/shares/pool-1", "file_ids": [upload.json()["file_id"]]}, + headers=headers, + ) + + +def _owners() -> set[str]: + rows = get_context().catalog.reservations_for_pool("1", now=datetime.now(UTC)) + return {str(r["owner"]) for r in rows} + + +def test_share_push_refused_booking_is_409_and_registers_no_plan( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + # The preview passes (budget free) but a concurrent start wins the booking. + def _refuse(*_: object, **__: object) -> None: + raise ReservationError("pool 1 cannot reserve capacity") + + monkeypatch.setattr("openblade.api.routes_upload.book_plan_capacity", _refuse) + plans_before = set(ArchivePlanStore) + + response = _push(client) + + assert response.status_code == 409 + assert set(ArchivePlanStore) == plans_before + + +def test_share_push_start_failure_releases_booking_and_plan( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + def _raise(**_: object) -> None: + raise _Boom("start failed") + + monkeypatch.setattr("openblade.api.routes_upload.start_ingest_job", _raise) + plans_before = set(ArchivePlanStore) + + with pytest.raises(_Boom): + _push(client) + + assert set(ArchivePlanStore) == plans_before + assert _owners() == set() + + +def test_share_push_raising_cleanup_keeps_start_error_and_unregisters( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + def _raise(**_: object) -> None: + raise _Boom("start failed") + + def _release_raises(*_: object) -> None: + raise RuntimeError("release exploded") + + monkeypatch.setattr("openblade.api.routes_upload.start_ingest_job", _raise) + monkeypatch.setattr("openblade.api.routes_upload.release_plan_capacity", _release_raises) + plans_before = set(ArchivePlanStore) + + with pytest.raises(_Boom): + _push(client) + assert set(ArchivePlanStore) == plans_before diff --git a/tests/unit/test_capacity_start_safety.py b/tests/unit/test_capacity_start_safety.py new file mode 100644 index 0000000..e0da06c --- /dev/null +++ b/tests/unit/test_capacity_start_safety.py @@ -0,0 +1,141 @@ +"""Start-path capacity safety: pool match, booking before registration, rollback.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from httpx import Response + +from openblade.api.main import app +from openblade.bootstrap import create_context, get_context, reset_context +from openblade.config import OpenBladeConfig +from openblade.nas.ingest import ArchivePlanStore, get_archive_plan + +client = TestClient(app) + + +@pytest.fixture(autouse=True) +def _context(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + reset_context(create_context(OpenBladeConfig(db_url=f"sqlite:///{tmp_path / 's.db'}"))) + for pool in ("pool-1", "pool-2"): + created = client.post("/nas/pools", json={"id": pool, "name": pool}) + assert created.status_code in {200, 201} + monkeypatch.setattr("openblade.api.nas_config.run_ingest_job", lambda *a, **k: None) + + +def _preview(pool: str | None) -> dict[str, object]: + body: dict[str, object] = { + "files": ["a.bin"], + "file_sizes": {"a.bin": 1_000}, + "ingest_mode": "source_stream", + } + if pool is None: + body.update({"available_tapes": ["T1"], "tape_capacities": {"T1": 10**12}}) + else: + body["pool"] = pool + plan: dict[str, object] = client.post("/nas/archive-plan", json=body).json() + assert plan["is_safe_to_enqueue"], plan + return plan + + +def _start(plan_id: object, pool_id: str | None) -> Response: + body = {"plan_id": plan_id, "dataset_name": "ds", "auto_clean_drives": False} + return client.post("/nas/ingest/start", json={**body, "pool_id": pool_id}) + + +def _owners(pool: str) -> set[str]: + rows = get_context().catalog.reservations_for_pool(pool, now=datetime.now(UTC)) + return {str(r["owner"]) for r in rows} + + +def test_start_into_a_different_pool_is_409_and_books_nothing() -> None: + plan = _preview("pool-1") + + response = _start(plan["plan_id"], "pool-2") + + assert response.status_code == 409 + assert response.json()["detail"] == "pool_id does not match plan pool" + assert _owners("pool-1") == set() + assert _owners("pool-2") == set() + + +def test_poolless_preview_cannot_start_into_a_pool() -> None: + plan = _preview(None) + + response = _start(plan["plan_id"], "pool-1") + + assert response.status_code == 409 + assert _owners("pool-1") == set() + + +def test_matching_pool_starts_and_dataset_uses_the_plan_pool() -> None: + from openblade.api.nas_config import get_nas_service + + plan = _preview("pool-1") + + response = _start(plan["plan_id"], "pool-1") + + assert response.status_code == 200, response.text + dataset_id = response.json()["dataset_id"] + dataset = get_nas_service(get_context().catalog).get_dataset(dataset_id) + assert dataset is not None and dataset.pool_id == "pool-1" + assert _owners("pool-1") == {plan["plan_id"]} + assert get_archive_plan(str(plan["plan_id"])) is not None + + +class _Boom(RuntimeError): + pass + + +def test_start_failure_after_booking_releases_booking_and_plan( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def _raise(**_: object) -> None: + raise _Boom("start failed") + + monkeypatch.setattr("openblade.api.nas_config.start_ingest_job", _raise) + plan = _preview("pool-1") + + with pytest.raises(_Boom): + _start(plan["plan_id"], "pool-1") + + assert _owners("pool-1") == set() + assert get_archive_plan(str(plan["plan_id"])) is None + + +def test_failed_rollback_release_does_not_mask_the_start_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def _raise(**_: object) -> None: + raise _Boom("start failed") + + def _broken_release(_owner: str) -> None: + raise OSError("db gone") + + monkeypatch.setattr("openblade.api.nas_config.start_ingest_job", _raise) + monkeypatch.setattr(get_context().catalog, "delete_nas_reservations_for_owner", _broken_release) + plan = _preview("pool-1") + + with pytest.raises(_Boom): + _start(plan["plan_id"], "pool-1") + + +def test_raising_cleanup_keeps_start_error_and_still_unregisters( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def _raise(**_: object) -> None: + raise _Boom("start failed") + + def _release_raises(*_: object) -> None: + raise RuntimeError("release exploded") + + monkeypatch.setattr("openblade.api.nas_config.start_ingest_job", _raise) + monkeypatch.setattr("openblade.api.nas_config.release_plan_capacity", _release_raises) + plan = _preview("pool-1") + + with pytest.raises(_Boom): + _start(plan["plan_id"], "pool-1") + assert plan["plan_id"] not in ArchivePlanStore diff --git a/tests/unit/test_cli_backend_selection.py b/tests/unit/test_cli_backend_selection.py index a7aa490..7a51fa6 100644 --- a/tests/unit/test_cli_backend_selection.py +++ b/tests/unit/test_cli_backend_selection.py @@ -28,6 +28,7 @@ def _clear_backend_env(monkeypatch: pytest.MonkeyPatch) -> None: for name in ( "OPENBLADE_BACKEND", "OPENBLADE_REAL_HARDWARE_ENABLED", + "OPENBLADE_API_TOKEN", "OPENBLADE_DB_URL", "OPENBLADE_CACHE_DIR", "OPENBLADE_STAGING_DIR", @@ -45,6 +46,7 @@ def test_default_config_honours_real_backend_env(monkeypatch: pytest.MonkeyPatch """The bug: this used to come back MOCK, silently simulating a real library.""" monkeypatch.setenv("OPENBLADE_BACKEND", "real") monkeypatch.setenv("OPENBLADE_REAL_HARDWARE_ENABLED", "true") + monkeypatch.setenv("OPENBLADE_API_TOKEN", "test-token") # real mode now requires an API token config = cli_main._default_config() @@ -66,6 +68,7 @@ def test_mock_config_pins_mock_even_under_real_env(monkeypatch: pytest.MonkeyPat """`openblade mock init` must never reach for a real changer.""" monkeypatch.setenv("OPENBLADE_BACKEND", "real") monkeypatch.setenv("OPENBLADE_REAL_HARDWARE_ENABLED", "true") + monkeypatch.setenv("OPENBLADE_API_TOKEN", "test-token") # real mode now requires an API token config = cli_main._mock_config() @@ -246,6 +249,7 @@ def test_mock_subcommands_never_reach_a_real_library(monkeypatch: pytest.MonkeyP """ monkeypatch.setenv("OPENBLADE_BACKEND", "real") monkeypatch.setenv("OPENBLADE_REAL_HARDWARE_ENABLED", "true") + monkeypatch.setenv("OPENBLADE_API_TOKEN", "test-token") # real mode now requires an API token captured: list[BackendMode] = [] monkeypatch.setattr( @@ -263,6 +267,7 @@ def test_plain_subcommands_still_honour_a_real_backend(monkeypatch: pytest.Monke """The guard above must not also neuter the non-mock commands.""" monkeypatch.setenv("OPENBLADE_BACKEND", "real") monkeypatch.setenv("OPENBLADE_REAL_HARDWARE_ENABLED", "true") + monkeypatch.setenv("OPENBLADE_API_TOKEN", "test-token") # real mode now requires an API token captured: list[BackendMode] = [] monkeypatch.setattr( diff --git a/tests/unit/test_config_real_backend_gate.py b/tests/unit/test_config_real_backend_gate.py new file mode 100644 index 0000000..5f77e26 --- /dev/null +++ b/tests/unit/test_config_real_backend_gate.py @@ -0,0 +1,54 @@ +"""Real backend needs a third signal (API token); simulator alias; unknown warns.""" + +from __future__ import annotations + +import logging + +import pytest + +from openblade.config import BackendMode, load_config +from openblade.config_validation import BLOCKING, validate_config +from openblade.domain.errors import RealHardwareDisabledError + +REAL = {"OPENBLADE_BACKEND": "real", "OPENBLADE_REAL_HARDWARE_ENABLED": "true"} + + +def _env(monkeypatch: pytest.MonkeyPatch, **env: str) -> None: + for name in ("OPENBLADE_API_TOKEN", "OPENBLADE_API_TOKEN_FILE"): + monkeypatch.delenv(name, raising=False) + for name, value in env.items(): + monkeypatch.setenv(name, value) + + +def test_load_config_real_enabled_without_api_token_raises( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _env(monkeypatch, **REAL) + with pytest.raises(RealHardwareDisabledError, match="OPENBLADE_API_TOKEN"): + load_config() + + +def test_load_config_real_enabled_with_api_token_loads(monkeypatch: pytest.MonkeyPatch) -> None: + _env(monkeypatch, **REAL, OPENBLADE_API_TOKEN="t" * 32) + assert load_config().backend == BackendMode.REAL + + +def test_validate_config_blocks_real_enabled_without_api_token() -> None: + findings = validate_config(dict(REAL)) + assert any( + f.code == "real_backend_requires_api_token" and f.severity == BLOCKING for f in findings + ) + + +def test_simulator_is_an_alias_of_mock(monkeypatch: pytest.MonkeyPatch) -> None: + _env(monkeypatch, OPENBLADE_BACKEND="simulator") + assert load_config().backend == BackendMode.MOCK + + +def test_unknown_backend_warns_and_uses_mock( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + _env(monkeypatch, OPENBLADE_BACKEND="reall") + with caplog.at_level(logging.WARNING, logger="openblade.config"): + assert load_config().backend == BackendMode.MOCK + assert "Unrecognised OPENBLADE_BACKEND" in caplog.text diff --git a/tests/unit/test_config_validation.py b/tests/unit/test_config_validation.py index a1bb511..1075d58 100644 --- a/tests/unit/test_config_validation.py +++ b/tests/unit/test_config_validation.py @@ -6,10 +6,13 @@ from __future__ import annotations +import pytest + from openblade.config_validation import ( BLOCKING, DEFAULT_SERVICE_PASSWORD, DEFAULT_SERVICE_TOKEN, + WARNING, Finding, is_deployable, validate_config, @@ -42,7 +45,7 @@ def test_production_blocks_default_admin_password_value() -> None: { "OPENBLADE_ENV": "production", "OPENBLADE_ADMIN_PASSWORD": "password", # the insecure default value - "OPENBLADE_SERVICE_PASSWORD": "x", + "OPENBLADE_SERVICE_PASSWORD": "x-strong-service-pw", "OPENBLADE_SERVICE_TOKEN": "strong", "OPENBLADE_DB_URL": "sqlite:////data/openblade.db", } @@ -54,8 +57,8 @@ def test_production_blocks_default_service_token() -> None: findings = validate_config( { "OPENBLADE_ENV": "production", - "OPENBLADE_ADMIN_PASSWORD": "s3cret", - "OPENBLADE_SERVICE_PASSWORD": "s3cret", + "OPENBLADE_ADMIN_PASSWORD": "s3cret-long-enough", + "OPENBLADE_SERVICE_PASSWORD": "s3cret-long-enough", "OPENBLADE_SERVICE_TOKEN": DEFAULT_SERVICE_TOKEN, "OPENBLADE_DB_URL": "sqlite:////data/openblade.db", } @@ -67,7 +70,7 @@ def test_production_blocks_default_service_password_value() -> None: findings = validate_config( { "OPENBLADE_ENV": "production", - "OPENBLADE_ADMIN_PASSWORD": "s3cret", + "OPENBLADE_ADMIN_PASSWORD": "s3cret-long-enough", "OPENBLADE_SERVICE_PASSWORD": DEFAULT_SERVICE_PASSWORD, # the shipped default 'service123' "OPENBLADE_SERVICE_TOKEN": "strong", "OPENBLADE_DB_URL": "sqlite:////data/openblade.db", @@ -81,8 +84,9 @@ def test_production_deployable_when_secrets_set() -> None: findings = validate_config( { "OPENBLADE_ENV": "production", - "OPENBLADE_ADMIN_PASSWORD": "s3cret-admin", - "OPENBLADE_SERVICE_PASSWORD": "s3cret-svc", + "OPENBLADE_API_TOKEN": "api", # production requires native API auth + "OPENBLADE_ADMIN_PASSWORD": "s3cret-admin-long", + "OPENBLADE_SERVICE_PASSWORD": "s3cret-svc-long-pw", "OPENBLADE_SERVICE_TOKEN": "0a1b2c3d-strong-token", "OPENBLADE_DB_URL": "sqlite:////data/openblade.db", } @@ -94,8 +98,8 @@ def test_webservices_transport_requires_scalar_url_in_production() -> None: findings = validate_config( { "OPENBLADE_ENV": "production", - "OPENBLADE_ADMIN_PASSWORD": "a", - "OPENBLADE_SERVICE_PASSWORD": "b", + "OPENBLADE_ADMIN_PASSWORD": "a-strong-admin-pw", + "OPENBLADE_SERVICE_PASSWORD": "b-strong-service-pw", "OPENBLADE_SERVICE_TOKEN": "t", "OPENBLADE_DB_URL": "sqlite:////data/db", "OPENBLADE_ROBOTICS_TRANSPORT": "webservices", # no OPENBLADE_SCALAR_URL @@ -108,8 +112,9 @@ def test_debug_logging_in_production_is_a_warning() -> None: findings = validate_config( { "OPENBLADE_ENV": "production", - "OPENBLADE_ADMIN_PASSWORD": "a", - "OPENBLADE_SERVICE_PASSWORD": "b", + "OPENBLADE_API_TOKEN": "api", # production requires native API auth + "OPENBLADE_ADMIN_PASSWORD": "a-strong-admin-pw", + "OPENBLADE_SERVICE_PASSWORD": "b-strong-service-pw", "OPENBLADE_SERVICE_TOKEN": "t", "OPENBLADE_DB_URL": "sqlite:////data/db", "OPENBLADE_LOG_LEVEL": "DEBUG", @@ -117,3 +122,212 @@ def test_debug_logging_in_production_is_a_warning() -> None: ) assert "debug_logging_in_production" in _codes(findings, "warning") assert is_deployable(findings) # warning does not block + + +# --- fail-closed startup gate (WS-harden-auth) --------------------------------- + +_SAFE_PROD = { + "OPENBLADE_ENV": "production", + "OPENBLADE_API_TOKEN": "api-strong", + "OPENBLADE_ADMIN_PASSWORD": "s3cret-long-enough", + "OPENBLADE_SERVICE_PASSWORD": "s3cret-long-enough", + "OPENBLADE_SERVICE_TOKEN": "strong", + "OPENBLADE_DB_URL": "sqlite:////data/openblade.db", + "OPENBLADE_CORS_ORIGINS": "http://lan-host:8091", +} + + +def test_startup_safe_production_starts() -> None: + from openblade.config_validation import enforce_startup_config + + enforce_startup_config(_SAFE_PROD) + + +@pytest.mark.parametrize( + ("override", "needle"), + [ + ({"OPENBLADE_API_TOKEN": ""}, "OPENBLADE_API_TOKEN"), + ({"OPENBLADE_ADMIN_PASSWORD": "password"}, "OPENBLADE_ADMIN_PASSWORD"), + ({"OPENBLADE_SERVICE_PASSWORD": "service123"}, "OPENBLADE_SERVICE_PASSWORD"), + ({"OPENBLADE_SERVICE_TOKEN": DEFAULT_SERVICE_TOKEN}, "OPENBLADE_SERVICE_TOKEN"), + ({"OPENBLADE_CORS_ORIGINS": "http://a,*"}, "OPENBLADE_CORS_ORIGINS"), + ], +) +def test_startup_production_refuses_each_open_gate(override: dict[str, str], needle: str) -> None: + from openblade.config_validation import ConfigurationError, enforce_startup_config + + with pytest.raises(ConfigurationError) as exc: + enforce_startup_config({**_SAFE_PROD, **override}) + assert needle in str(exc.value) + # Never echoes the configured secret values. + assert "s3cret" not in str(exc.value) and "api-strong" not in str(exc.value) + + +def test_startup_api_token_file_satisfies_gate() -> None: + from openblade.config_validation import enforce_startup_config + + enforce_startup_config( + {**_SAFE_PROD, "OPENBLADE_API_TOKEN": "", "OPENBLADE_API_TOKEN_FILE": "/run/t"} + ) + + +def test_startup_development_banner_lists_every_open_gate() -> None: + from openblade.config_validation import enforce_startup_config, startup_banner + + banner = startup_banner( + enforce_startup_config({"OPENBLADE_ENV": "development", "OPENBLADE_CORS_ORIGINS": "*"}) + ) + for line in ( + "NATIVE API OPEN", + "WEAK AML ADMIN PASSWORD", + "WEAK AML SERVICE PASSWORD", + "DEFAULT SERVICE TOKEN", + "CORS WILDCARD", + ): + assert line in banner + + +@pytest.mark.parametrize( + ("func", "var"), + [ + ("_get_admin_password", "OPENBLADE_ADMIN_PASSWORD"), + ("_get_service_password", "OPENBLADE_SERVICE_PASSWORD"), + ], +) +def test_aml_default_password_refused_in_production( + monkeypatch: pytest.MonkeyPatch, func: str, var: str +) -> None: + from openblade.api import aml_state + from openblade.config_validation import ConfigurationError + + monkeypatch.setenv("OPENBLADE_ENV", "production") + monkeypatch.delenv(var, raising=False) + with pytest.raises(ConfigurationError): + getattr(aml_state, func)() + monkeypatch.setenv("OPENBLADE_ENV", "development") + assert getattr(aml_state, func)() # dev keeps the warning-only behaviour + + +def test_api_startup_hook_refuses_production_without_api_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import asyncio + + from openblade.api import main + from openblade.config_validation import ConfigurationError + + for key, value in _SAFE_PROD.items(): + monkeypatch.setenv(key, value) + monkeypatch.delenv("OPENBLADE_API_TOKEN") + monkeypatch.delenv("OPENBLADE_API_TOKEN_FILE", raising=False) + with pytest.raises(ConfigurationError, match="OPENBLADE_API_TOKEN"): + asyncio.run(main.initialize_aml_state()) + + +def test_production_insecure_session_cookies_warn() -> None: + findings = validate_config({**_SAFE_PROD, "OPENBLADE_WEB_SECURE_COOKIES": "false"}) + assert "insecure_session_cookies" in _codes(findings, WARNING) + assert "insecure_session_cookies" not in _codes(validate_config(_SAFE_PROD)) + + +def test_web_flask_refuses_production_without_secret_key( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from openblade.web_flask import app as web_app + + monkeypatch.setenv("OPENBLADE_ENV", "production") + monkeypatch.delenv("FLASK_SECRET_KEY", raising=False) + with pytest.raises(RuntimeError, match="FLASK_SECRET_KEY"): + web_app.create_app() + + +def test_web_flask_production_cookies_secure_by_default( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from openblade.web_flask import app as web_app + + monkeypatch.setenv("OPENBLADE_ENV", "production") + monkeypatch.setenv("FLASK_SECRET_KEY", "k" * 32) + monkeypatch.delenv("OPENBLADE_WEB_SECURE_COOKIES", raising=False) + assert web_app.create_app().config["SESSION_COOKIE_SECURE"] is True + + +def test_dev_db_path_finding_redacts_url_credentials() -> None: + """WARNING findings are logged verbatim by the startup banner: no secrets.""" + findings = validate_config( + { + "OPENBLADE_ENV": "production", + "OPENBLADE_DB_URL": "postgresql://ob:S3cretPassw0rd@db.lan:5432/tmp/openblade", + } + ) + msgs = [f.message for f in findings if f.code == "dev_db_path"] + assert len(msgs) == 1 + assert "S3cretPassw0rd" not in msgs[0] + assert "ob:" not in msgs[0] + assert "postgresql://***@db.lan:5432/tmp/openblade" in msgs[0] + assert all("S3cretPassw0rd" not in f.message for f in findings) + + +def test_redact_url_drops_query_and_handles_plain_paths() -> None: + from openblade.config_validation import redact_url + + assert redact_url("postgresql://u:p@h/db?password=x") == "postgresql://***@h/db" + assert redact_url("sqlite:////tmp/x.db") == "sqlite:////tmp/x.db" + + +@pytest.mark.parametrize( + "value", + [" ", " \t", "short-pw", "password", "service123", " service123 "], + ids=["space", "whitespace", "short", "admin-default", "svc-default", "padded-default"], +) +@pytest.mark.parametrize( + ("var", "code"), + [ + ("OPENBLADE_ADMIN_PASSWORD", "unsafe_admin_password"), + ("OPENBLADE_SERVICE_PASSWORD", "missing_service_password"), + ], +) +def test_production_blocks_weak_passwords(value: str, var: str, code: str) -> None: + env = dict(_SAFE_PROD) + env[var] = value + findings = validate_config(env) + assert code in _codes(findings, BLOCKING) + assert not is_deployable(findings) + + +@pytest.mark.parametrize("var", ["OPENBLADE_ADMIN_PASSWORD", "OPENBLADE_SERVICE_PASSWORD"]) +def test_development_warns_on_short_password(var: str) -> None: + findings = validate_config({"OPENBLADE_ENV": "development", var: "short-pw"}) + assert is_deployable(findings) + msgs = [f.message for f in findings if f.severity == WARNING and var in f.message] + assert msgs and "shorter than 12" in msgs[0] and "short-pw" not in msgs[0] + + +def test_twelve_char_password_is_accepted() -> None: + from openblade.config_validation import password_problem + + assert password_problem("abcdefghijkl") is None + assert password_problem("abcdefghijk") is not None + + +@pytest.mark.parametrize( + ("getter", "var"), + [ + ("_get_admin_password", "OPENBLADE_ADMIN_PASSWORD"), + ("_get_service_password", "OPENBLADE_SERVICE_PASSWORD"), + ], +) +@pytest.mark.parametrize("value", [" ", "short-pw", "password", "service123"]) +def test_aml_password_getters_refuse_weak_values_in_production( + monkeypatch: pytest.MonkeyPatch, getter: str, var: str, value: str +) -> None: + from openblade.api import aml_state + from openblade.config_validation import ConfigurationError + + monkeypatch.setenv("OPENBLADE_ENV", "production") + monkeypatch.setenv(var, value) + with pytest.raises(ConfigurationError) as exc: + getattr(aml_state, getter)() + assert value.strip() == "" or value not in str(exc.value) + monkeypatch.setenv(var, "a-strong-password-1") + assert getattr(aml_state, getter)() == "a-strong-password-1" diff --git a/tests/unit/test_format_scratch_allowlist.py b/tests/unit/test_format_scratch_allowlist.py new file mode 100644 index 0000000..1a1694e --- /dev/null +++ b/tests/unit/test_format_scratch_allowlist.py @@ -0,0 +1,83 @@ +"""On a REAL backend FormatService formats only OPENBLADE_SCRATCH_BARCODES tapes. + +Fakes only: ``real_backend=True`` is a constructor flag; no device, no enable flag. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import pytest + +from openblade.domain.errors import FormatRequiresConfirmationError +from openblade.domain.media_allowlist import is_scratch_barcode, parse_barcodes +from openblade.jobs.format import FormatService + +SCRATCH = "SCR001L9" +DATA = "DATA01L9" + + +def _service(*, real: bool | None, allowlist: list[str] | None) -> FormatService: + inventory = SimpleNamespace( + slots=[SimpleNamespace(barcode=SCRATCH), SimpleNamespace(barcode=DATA)], drives=[] + ) + library = SimpleNamespace(inventory=lambda: inventory) + catalog = SimpleNamespace(save_safety_token=lambda token: None) + fake: Any = (catalog, library, SimpleNamespace()) + return FormatService(*fake, real_backend=real, scratch_barcodes=allowlist) + + +def test_real_backend_dry_run_on_unlisted_tape_is_refused() -> None: + with pytest.raises(FormatRequiresConfirmationError, match="OPENBLADE_SCRATCH_BARCODES"): + _service(real=True, allowlist=[SCRATCH]).dry_run(DATA) + + +def test_real_backend_confirm_on_unlisted_tape_is_refused_before_token_lookup() -> None: + # The fake catalog has no get_safety_token: reaching it would AttributeError. + with pytest.raises(FormatRequiresConfirmationError, match="OPENBLADE_SCRATCH_BARCODES"): + _service(real=True, allowlist=[SCRATCH]).confirm(DATA, "any-token") + + +def test_real_backend_empty_allowlist_refuses_everything() -> None: + with pytest.raises(FormatRequiresConfirmationError): + _service(real=True, allowlist=[]).dry_run(SCRATCH) + + +def test_real_backend_allowlisted_tape_dry_run_is_allowed() -> None: + plan, token = _service(real=True, allowlist=[SCRATCH]).dry_run(SCRATCH) + assert plan.target == SCRATCH + assert token.target_barcode == SCRATCH + + +def test_simulator_backend_is_not_allowlist_gated() -> None: + plan, _ = _service(real=False, allowlist=[]).dry_run(DATA) + assert plan.target == DATA + + +def test_default_reads_backend_and_allowlist_from_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENBLADE_BACKEND", "real") + monkeypatch.delenv("OPENBLADE_REAL_HARDWARE_ENABLED", raising=False) + monkeypatch.setenv("OPENBLADE_SCRATCH_BARCODES", f" {SCRATCH.lower()} ") + service = _service(real=None, allowlist=None) + with pytest.raises(FormatRequiresConfirmationError): + service.dry_run(DATA) + assert service.dry_run(SCRATCH)[0].target == SCRATCH + + +def test_is_scratch_barcode_normalises_and_fails_closed() -> None: + assert is_scratch_barcode("scr001l9", parse_barcodes("SCR001L9,X")) + assert not is_scratch_barcode(SCRATCH, []) + + +def test_real_backend_padded_lowercase_barcode_is_normalised_and_accepted() -> None: + inventory = SimpleNamespace(slots=[SimpleNamespace(barcode="ABC123")], drives=[]) + library = SimpleNamespace(inventory=lambda: inventory) + catalog = SimpleNamespace(save_safety_token=lambda token: None) + fake: Any = (catalog, library, SimpleNamespace()) + service = FormatService(*fake, real_backend=True, scratch_barcodes=["ABC123"]) + + plan, token = service.dry_run(" abc123 ") + + assert plan.target == "ABC123" + assert token.target_barcode == "ABC123" diff --git a/tests/unit/test_format_token_single_use.py b/tests/unit/test_format_token_single_use.py new file mode 100644 index 0000000..3b12d05 --- /dev/null +++ b/tests/unit/test_format_token_single_use.py @@ -0,0 +1,185 @@ +"""Format safety tokens are single-use and consumed BEFORE the destructive job. + +Fakes only: the format job is replaced, no device is touched. +""" + +from __future__ import annotations + +import threading +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from openblade.catalog.db import get_session, init_db +from openblade.catalog.repository import CatalogRepository +from openblade.domain.errors import FormatRequiresConfirmationError +from openblade.domain.models import OperationResult +from openblade.domain.policies import SafetyToken +from openblade.jobs import format as format_module +from openblade.jobs.format import FormatService + +BARCODE = "SCR001L9" + + +class _Catalog: + """Delegates token methods to a real repository; stubs the rest.""" + + def __init__(self, repo: CatalogRepository, barrier: threading.Barrier | None) -> None: + self.repo = repo + self.barrier = barrier + self.session = SimpleNamespace(commit=lambda: None) + self.fail_delete = False + + def get_safety_token(self, value: str) -> SafetyToken | None: + token = self.repo.get_safety_token(value) + if self.barrier is not None: + # Both confirms have read the (valid) token before either proceeds. + self.barrier.wait(timeout=5) + return token + + def delete_safety_token(self, value: str) -> bool: + if self.fail_delete: + return False + return self.repo.delete_safety_token(value) + + def add_cartridge(self, barcode: str) -> Any: + return SimpleNamespace(formatted=False, used_bytes=1) + + +@pytest.fixture +def repo(tmp_path: Path) -> CatalogRepository: + init_db(f"sqlite:///{tmp_path / 'catalog.db'}") + return CatalogRepository(get_session()) + + +@pytest.fixture +def job_calls(monkeypatch: pytest.MonkeyPatch) -> list[str]: + calls: list[str] = [] + + def fake_job(barcode: str, *_: object, **__: object) -> OperationResult: + calls.append(barcode) + return OperationResult(success=True, message="formatted (fake)") + + monkeypatch.setattr(format_module, "run_format_job", fake_job) + return calls + + +def _service(catalog: _Catalog) -> FormatService: + return FormatService(catalog, SimpleNamespace(), SimpleNamespace()) # type: ignore[arg-type] # fakes stand in for typed backends + + +def _armed_token(repo: CatalogRepository) -> SafetyToken: + token = SafetyToken.generate("format", BARCODE) + repo.save_safety_token(token) + return token + + +def test_repository_delete_is_single_use(repo: CatalogRepository) -> None: + token = _armed_token(repo) + assert repo.delete_safety_token(token.token) is True + assert repo.delete_safety_token(token.token) is False + + +def test_sequential_second_confirm_is_refused( + repo: CatalogRepository, job_calls: list[str] +) -> None: + token = _armed_token(repo) + service = _service(_Catalog(repo, None)) + service.confirm(BARCODE, token.token) + with pytest.raises(FormatRequiresConfirmationError): + service.confirm(BARCODE, token.token) + assert job_calls == [BARCODE] + + +def test_concurrent_confirms_with_one_token_run_the_job_once( + repo: CatalogRepository, job_calls: list[str] +) -> None: + token = _armed_token(repo) + service = _service(_Catalog(repo, threading.Barrier(2))) + outcomes: list[str] = [] + + def confirm() -> None: + try: + service.confirm(BARCODE, token.token) + outcomes.append("ok") + except FormatRequiresConfirmationError: + outcomes.append("refused") + + threads = [threading.Thread(target=confirm) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + assert sorted(outcomes) == ["ok", "refused"] + assert job_calls == [BARCODE] + + +def test_two_repositories_on_one_file_claim_a_token_exactly_once(tmp_path: Path) -> None: + """Cross-process claim: independent engines share only the sqlite FILE. + + Each repository has its own engine, connection pool and ``_lock``, so the + in-process RLock cannot serialise them; only the database lock can. + """ + from sqlalchemy import create_engine + from sqlalchemy.orm import Session + + from openblade.catalog.db import _sqlite_connect_args + + url = f"sqlite:///{tmp_path / 'shared.db'}" + init_db(url) + engines = [create_engine(url, connect_args=_sqlite_connect_args(url)) for _ in range(2)] + repos = [CatalogRepository(Session(bind=e, expire_on_commit=False)) for e in engines] + assert repos[0]._lock is not repos[1]._lock + try: + for _ in range(25): + token = _armed_token(repos[0]) + barrier = threading.Barrier(2) + claims: list[bool] = [] + + def claim( + r: CatalogRepository, + value: str = token.token, + barrier: threading.Barrier = barrier, + claims: list[bool] = claims, + ) -> None: + barrier.wait(timeout=10) + claims.append(r.delete_safety_token(value)) + + threads = [threading.Thread(target=claim, args=(r,)) for r in repos] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + assert sorted(claims) == [False, True] + finally: + for r in repos: + r.session.close() + for e in engines: + e.dispose() + + +def test_failed_delete_refuses_and_never_runs_the_job( + repo: CatalogRepository, job_calls: list[str] +) -> None: + token = _armed_token(repo) + catalog = _Catalog(repo, None) + catalog.fail_delete = True + with pytest.raises(FormatRequiresConfirmationError): + _service(catalog).confirm(BARCODE, token.token) + assert job_calls == [] + + +def test_failed_job_does_not_rearm_the_token( + repo: CatalogRepository, monkeypatch: pytest.MonkeyPatch +) -> None: + token = _armed_token(repo) + + def boom(*_: object, **__: object) -> OperationResult: + raise RuntimeError("fake drive failure") + + monkeypatch.setattr(format_module, "run_format_job", boom) + with pytest.raises(RuntimeError): + _service(_Catalog(repo, None)).confirm(BARCODE, token.token) + assert repo.delete_safety_token(token.token) is False diff --git a/tests/unit/test_hardware_preflight.py b/tests/unit/test_hardware_preflight.py new file mode 100644 index 0000000..0eff347 --- /dev/null +++ b/tests/unit/test_hardware_preflight.py @@ -0,0 +1,190 @@ +"""Unit tests for tools.hardware.preflight — every check against fakes, no hardware.""" + +from __future__ import annotations + +from collections.abc import Mapping + +from tests.fakes.mtx import StatefulMtxRunner +from tools.hardware import preflight as pf + +CHANGER = "/dev/sg9" +DRIVES = ["/dev/nst0", "/dev/nst1", "/dev/nst2"] +SERIALS = {"/dev/nst0": "SN0", "/dev/nst1": "SN1", "/dev/nst2": "SN2"} + + +class FakeAppliance: + def __init__(self, code: int = 200) -> None: + self.code = code + self.calls: list[str] = [] + + def login(self) -> int: + self.calls.append("login") + return self.code + + def logout(self) -> int: + self.calls.append("logout") + return 200 + + +def _env(**over: str) -> dict[str, str]: + env = { + "OPENBLADE_CHANGER_DEVICE": CHANGER, + "OPENBLADE_DRIVE_DEVICES": ",".join(DRIVES), + "OPENBLADE_APPLIANCE_URL": "https://i3.example", + "OPENBLADE_APPLIANCE_USER": "ops", + "OPENBLADE_APPLIANCE_PASSWORD": "s3cret-pw", + "SACRIFICIAL_BARCODES": "AAA001L8,AAA002L8", + } + env.update(over) + return {k: v for k, v in env.items() if v != ""} + + +def _host( + env: Mapping[str, str] | None = None, + *, + serials: dict[str, str] | None = None, + appliance: FakeAppliance | None = None, +) -> tuple[pf.Host, StatefulMtxRunner]: + runner = StatefulMtxRunner( + CHANGER, + drives=3, + slots=10, + barcodes={1: "AAA001L8", 2: "AAA002L8", 3: "OTHER3L8"}, + drive_serials=serials if serials is not None else SERIALS, + ) + fake = appliance or FakeAppliance() + host = pf.Host( + env=env if env is not None else _env(), + runner=runner, + which=lambda t: f"/usr/bin/{t}", + exists=lambda p: True, + readable=lambda p: True, + by_id=lambda: [f"{d}|IBM-ULTRIUM-TD8_{s}-nst" for d, s in SERIALS.items()], + appliance=lambda _env: fake, + ) + return host, runner + + +def _by(checks: list[pf.Check], status: str) -> list[pf.Check]: + return [c for c in checks if c.status == status] + + +def test_happy_path_has_no_fail_or_warn_and_never_moves_media() -> None: + host, runner = _host() + checks = pf.run_preflight(host, "destructive") + assert _by(checks, "FAIL") == [] and _by(checks, "WARN") == [] + verbs = {c[3] for c in runner.calls if c[0] == "mtx"} + assert verbs == {"status"} # read-only: inquiry/listing only + + +def test_missing_env_fail_names_the_var() -> None: + host, _ = _host(_env(OPENBLADE_APPLIANCE_PASSWORD="")) + fails = _by(pf.check_env(host, "readonly"), "FAIL") + assert [c.name for c in fails] == ["env:OPENBLADE_APPLIANCE_PASSWORD"] + assert "OPENBLADE_APPLIANCE_PASSWORD" in fails[0].detail + + +def test_password_value_is_never_emitted() -> None: + host, _ = _host() + assert all("s3cret-pw" not in (c.detail + c.fix) for c in pf.run_preflight(host, "readonly")) + + +def test_allowlist_required_for_destructive_only() -> None: + host, _ = _host(_env(SACRIFICIAL_BARCODES="")) + assert _by(pf.run_preflight(host, "readonly"), "FAIL") == [] + assert {c.name for c in _by(pf.run_preflight(host, "destructive"), "FAIL")} == { + "env:SACRIFICIAL_BARCODES", + "allowlist", + } + + +def test_duplicate_serials_fail() -> None: + dup = {"/dev/nst0": "SN0", "/dev/nst1": "SN0", "/dev/nst2": "SN2"} + host, _ = _host(serials=dup) + fails = _by(pf.check_devices(host), "FAIL") + assert any(c.name == "serials" and "SN0" in c.detail for c in fails) + + +def test_missing_serial_and_unreadable_device_fail() -> None: + host, _ = _host(serials={"/dev/nst0": "SN0", "/dev/nst1": "SN1"}) + host.readable = lambda p: p != CHANGER + names = {c.name for c in _by(pf.check_devices(host), "FAIL")} + assert names == {"inquiry:/dev/nst2", f"device:{CHANGER}"} + + +def test_missing_by_id_link_fails() -> None: + host, _ = _host() + host.by_id = lambda: [] + assert {c.name for c in _by(pf.check_devices(host), "FAIL")} == { + "by-id:SN0", + "by-id:SN1", + "by-id:SN2", + } + + +def test_serial_map_must_agree_with_inquiry_order() -> None: + ok, _ = _host(_env(OPENBLADE_DRIVE_SERIAL_MAP="SN0:0,SN1:1,SN2:2")) + assert _by(pf.check_devices(ok), "FAIL") == [] + bad, _ = _host(_env(OPENBLADE_DRIVE_SERIAL_MAP="SN1:0,SN0:1,SN2:2")) + assert [c.name for c in _by(pf.check_devices(bad), "FAIL")] == ["serial-map"] + junk, _ = _host(_env(OPENBLADE_DRIVE_SERIAL_MAP="nonsense")) + assert [c.name for c in _by(pf.check_devices(junk), "FAIL")] == ["serial-map"] + + +def test_changer_drive_count_mismatch_fails() -> None: + host, _ = _host(_env(OPENBLADE_DRIVE_DEVICES="/dev/nst0,/dev/nst1")) + checks, status = pf.check_changer(host) + assert status is not None and checks[0].status == "FAIL" and "3 drives" in checks[0].detail + + +def test_allowlisted_barcode_loaded_fails() -> None: + host, runner = _host() + runner.drives[0] = ("AAA001L8", 1) + runner.slots[1] = None + _, status = pf.check_changer(host) + fails = _by(pf.check_allowlist(host, status, "destructive"), "FAIL") + assert any("loaded" in c.detail and "AAA001L8" in c.detail for c in fails) + + +def test_allowlisted_barcode_absent_from_inventory_fails() -> None: + host, _ = _host(_env(SACRIFICIAL_BARCODES="AAA001L8,ZZZ999L8")) + _, status = pf.check_changer(host) + fails = _by(pf.check_allowlist(host, status, "destructive"), "FAIL") + assert any("ZZZ999L8" in c.detail for c in fails) + + +def test_armed_env_fails() -> None: + host, _ = _host(_env(OPENBLADE_REAL_HARDWARE_ENABLED="true")) + assert pf.check_armed(host).status == "FAIL" + + +def test_armed_env_fails_an_otherwise_healthy_run() -> None: + # main() exits EXIT_FAIL iff any check FAILs; armed must be enough on its own. + host, _ = _host(_env(OPENBLADE_REAL_HARDWARE_ENABLED="true")) + + checks = pf.run_preflight(host, "readonly") + + assert [c.name for c in _by(checks, "FAIL")] == ["armed"] + + +def test_missing_tool_fails() -> None: + host, _ = _host() + host.which = lambda t: None if t == "mkltfs" else f"/usr/bin/{t}" + assert [c.name for c in _by(pf.check_tools(host), "FAIL")] == ["tool:mkltfs"] + + +def test_appliance_login_logout_and_insecure_warn() -> None: + fake = FakeAppliance() + host, _ = _host(_env(OPENBLADE_APPLIANCE_INSECURE="true"), appliance=fake) + checks = pf.check_appliance(host) + assert fake.calls == ["login", "logout"] + assert [c.status for c in checks] == ["WARN", "PASS"] + denied, _ = _host(appliance=FakeAppliance(401)) + assert pf.check_appliance(denied)[0].status == "FAIL" + + +def test_appliance_bad_scheme_is_a_curated_fail() -> None: + host, _ = _host(_env(OPENBLADE_APPLIANCE_URL="ftp://i3")) + host.appliance = pf._default_appliance + (check,) = pf.check_appliance(host) + assert check.status == "FAIL" and "http" in check.detail diff --git a/tests/unit/test_nas_ingest_format_safety.py b/tests/unit/test_nas_ingest_format_safety.py new file mode 100644 index 0000000..a27c6d6 --- /dev/null +++ b/tests/unit/test_nas_ingest_format_safety.py @@ -0,0 +1,72 @@ +"""Ingest must never format a tape on a real backend (reviewer BLOCKER, fleet4).""" + +from pathlib import Path +from typing import Any + +import pytest + +from openblade.bootstrap import get_context +from openblade.nas.ingest import register_archive_plan, start_ingest_job +from openblade.nas.types import DatasetStatus +from tests.unit.test_nas_ingest import _make_plan, _run_job, _setup_service + + +def _start(tmp_path: Path) -> tuple[Any, str]: + service, cache_root = _setup_service(tmp_path) + job = start_ingest_job( + plan=register_archive_plan(_make_plan(cache_root)), + dataset_name="dataset-a", + pool_id="pool-1", + nas_service=service, + cache_drive_id="cache-1", + ) + return service, job.job_id + + +def _record_format_calls(monkeypatch: pytest.MonkeyPatch) -> list[str]: + ltfs = get_context().ltfs + real_format = ltfs.format + calls: list[str] = [] + + def recording_format(barcode: str, confirmation: Any) -> Any: + calls.append(barcode) + return real_format(barcode, confirmation) + + monkeypatch.setattr(ltfs, "format", recording_format) + return calls + + +def test_real_backend_unformatted_tape_is_refused_and_never_formatted( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + service, job_id = _start(tmp_path) + calls = _record_format_calls(monkeypatch) + # Context is already a simulator; only the backend detection flips to REAL. + monkeypatch.setenv("OPENBLADE_BACKEND", "real") + monkeypatch.delenv("OPENBLADE_REAL_HARDWARE_ENABLED", raising=False) + + result = _run_job(service, job_id) + + assert calls == [] + assert result.status is DatasetStatus.FAILED + assert any( + "not LTFS-formatted" in error and "/ltfs/format" in error for error in result.errors + ), result.errors + assert all( + not record.status.name.startswith("OFFLINE") + for record in service.list_file_records(result.dataset_id) + ) + + +def test_simulator_unformatted_tape_is_auto_formatted( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + service, job_id = _start(tmp_path) + calls = _record_format_calls(monkeypatch) + monkeypatch.setenv("OPENBLADE_BACKEND", "mock") + + result = _run_job(service, job_id) + + assert result.status is DatasetStatus.ARCHIVED + assert len(calls) == 1 + assert get_context().ltfs.ensure_tape(calls[0]).formatted diff --git a/tests/unit/test_tapealert.py b/tests/unit/test_tapealert.py index beea932..83f6e10 100644 --- a/tests/unit/test_tapealert.py +++ b/tests/unit/test_tapealert.py @@ -240,6 +240,7 @@ def test_drive_health_cli_reports_unsupported_drives(monkeypatch: pytest.MonkeyP monkeypatch.setenv("OPENBLADE_BACKEND", "real") monkeypatch.setenv("OPENBLADE_REAL_HARDWARE_ENABLED", "true") + monkeypatch.setenv("OPENBLADE_API_TOKEN", "test-token") # real mode now requires an API token monkeypatch.setattr( fuse_and_health, "sg_inq", diff --git a/tests/unit/test_test_runner_gate.py b/tests/unit/test_test_runner_gate.py new file mode 100644 index 0000000..243bcbd --- /dev/null +++ b/tests/unit/test_test_runner_gate.py @@ -0,0 +1,71 @@ +"""The test-runner route is dev-only and never targets an off-host AML (SSRF).""" + +from __future__ import annotations + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from openblade.api import routes_test_runner +from openblade.api.routes_test_runner import aml_url_allowed + + +@pytest.fixture +def client() -> TestClient: + app = FastAPI() + app.include_router(routes_test_runner.router) + return TestClient(app) + + +@pytest.mark.parametrize( + ("env", "enabled"), + [ + ("development", None), + ("development", "false"), + ("production", "true"), + ], +) +def test_run_is_404_unless_enabled_outside_production( + client: TestClient, monkeypatch: pytest.MonkeyPatch, env: str, enabled: str | None +) -> None: + monkeypatch.setenv("OPENBLADE_ENV", env) + if enabled is None: + monkeypatch.delenv("OPENBLADE_ENABLE_TEST_RUNNER", raising=False) + else: + monkeypatch.setenv("OPENBLADE_ENABLE_TEST_RUNNER", enabled) + resp = client.post("/api/test-runner/run", json={}) + assert resp.status_code == 404 + + +@pytest.mark.parametrize( + "url", + [ + "http://10.0.0.5:8000", + "http://169.254.169.254/latest", + "http://localhost.evil.example", + "http://127.0.0.1@evil.example", + "file:///etc/passwd", + "gopher://localhost", + ], +) +def test_run_rejects_off_host_aml_url_when_enabled( + client: TestClient, monkeypatch: pytest.MonkeyPatch, url: str +) -> None: + monkeypatch.setenv("OPENBLADE_ENV", "development") + monkeypatch.setenv("OPENBLADE_ENABLE_TEST_RUNNER", "true") + resp = client.post("/api/test-runner/run", json={"i3_aml_url": url}) + assert resp.status_code == 400 + assert url not in resp.text + + +@pytest.mark.parametrize( + "url", + [ + "http://localhost:8000", + "http://127.0.0.1:8010", + "http://[::1]:8000", + "https://host.docker.internal:8443", + ], +) +def test_loopback_and_docker_host_urls_allowed(url: str) -> None: + assert aml_url_allowed(url) diff --git a/tools/ci_ownership.toml b/tools/ci_ownership.toml index 024b683..bb6f671 100644 --- a/tools/ci_ownership.toml +++ b/tools/ci_ownership.toml @@ -62,6 +62,11 @@ categories = ["compat"] glob = "tools/compat_capture*" categories = ["compat"] +# Hardware lane tooling (preflight, allowlist, junit gate) is unit-tested with fakes. +[[rule]] +glob = "tools/hardware/*" +categories = ["unit"] + [[rule]] glob = "tests/fixtures/*" categories = ["unit", "compat"] diff --git a/tools/hardware/allowlist.py b/tools/hardware/allowlist.py index 207a405..f2b54a6 100644 --- a/tools/hardware/allowlist.py +++ b/tools/hardware/allowlist.py @@ -18,54 +18,22 @@ import argparse import os -import re import sys from collections.abc import Sequence +# Parsing lives in the domain so the service-side gate and this CI tool share it. +from openblade.domain.media_allowlist import ( + AllowlistError, + check_subset, + format_barcodes, + parse_barcodes, +) + EXIT_OK = 0 EXIT_VIOLATION = 2 -_SEPARATORS = re.compile(r"[\s,]+") - - -class AllowlistError(Exception): - """The requested barcodes are not safe to use destructively.""" - - -def parse_barcodes(raw: str | None) -> list[str]: - """Split a comma/whitespace list into upper-cased, de-duplicated barcodes.""" - seen: dict[str, None] = {} - for token in _SEPARATORS.split(raw or ""): - if token: - seen.setdefault(token.strip().upper(), None) - return list(seen) - -def format_barcodes(barcodes: Sequence[str]) -> str: - """The normalized stdout form: comma-joined, no spaces.""" - return ",".join(barcodes) - - -def check_subset(requested: Sequence[str], allowlist: Sequence[str]) -> list[str]: - """Return ``requested`` if it is a non-empty subset of a non-empty allowlist. - - Raises :class:`AllowlistError` naming the offenders otherwise. - """ - if not allowlist: - raise AllowlistError( - "SACRIFICIAL_BARCODES is empty: no tape is approved for destructive use (fail closed)." - ) - if not requested: - raise AllowlistError("No barcodes requested (pass --barcodes or set BARCODES).") - allowed = set(allowlist) - offenders = [b for b in requested if b not in allowed] - if offenders: - raise AllowlistError( - "Barcodes outside the sacrificial allowlist: " - + ", ".join(offenders) - + ". Refusing before touching hardware." - ) - return list(requested) +__all__ = ["AllowlistError", "check_subset", "format_barcodes", "main", "parse_barcodes"] def main(argv: Sequence[str] | None = None) -> int: diff --git a/tools/hardware/preflight.py b/tools/hardware/preflight.py new file mode 100644 index 0000000..92a2751 --- /dev/null +++ b/tools/hardware/preflight.py @@ -0,0 +1,412 @@ +"""Day-one appliance preflight: is this runner host ready for the hardware lanes? + +READ-ONLY on hardware: it runs inquiries and listings only (``sg_inq``, ``mtx status``, +an appliance login/logout) and never moves, loads, formats or erases media. It must +never run armed, so it does not need (and FAILs on) ``OPENBLADE_REAL_HARDWARE_ENABLED``. + +Usage:: + + python -m tools.hardware.preflight --lane readonly|destructive [--json] + +Exit 0 when no check FAILs, 1 otherwise. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sys +import urllib.error +import urllib.request +from collections.abc import Callable, Mapping, Sequence +from dataclasses import asdict, dataclass, field +from typing import Literal, Protocol + +from openblade.config import parse_drive_serial_map +from openblade.domain.errors import DriveCorrelationError +from openblade.hardware.appliance_client import ApplianceClient, ApplianceResponse +from openblade.hardware.mtx import MtxStatus, parse_mtx_status +from openblade.hardware.runner import SafeRunner +from openblade.hardware.sg import parse_sg_inq +from tools.hardware.allowlist import parse_barcodes + +Status = Literal["PASS", "WARN", "FAIL"] +Lane = Literal["readonly", "destructive"] + +EXIT_OK = 0 +EXIT_FAIL = 1 + +REQUIRED_TOOLS: tuple[str, ...] = ("sg_inq", "lsscsi", "mtx", "ltfs", "mkltfs") +BY_ID_DIR = "/dev/tape/by-id" + + +@dataclass(frozen=True) +class Check: + name: str + status: Status + detail: str + fix: str = "" + + +class ApplianceProbe(Protocol): + def login(self) -> int: ... + def logout(self) -> int: ... + + +class PreflightError(Exception): + """A curated, operator-facing failure (never carries raw upstream text).""" + + +class _ApplianceProbe: + """Login/logout probe over the shared read-only ApplianceClient.""" + + def __init__(self, base_url: str, user: str, password: str, *, insecure: bool) -> None: + # Validate at the point of use: the URL arrives from runner env, not an API. + if not base_url.startswith(("https://", "http://")): + raise PreflightError("OPENBLADE_APPLIANCE_URL must start with http:// or https://") + self._client = ApplianceClient( + base_url=base_url, user=user, password=password, insecure=insecure, timeout_s=15.0 + ) + + def _call(self, op: Callable[[], ApplianceResponse]) -> int: + try: + return op().status + except (urllib.error.URLError, OSError, TimeoutError) as exc: + raise PreflightError(f"appliance unreachable ({type(exc).__name__})") from exc + + def login(self) -> int: + return self._call(self._client.login) + + def logout(self) -> int: + return self._call(self._client.logout) + + +def _default_appliance(env: Mapping[str, str]) -> ApplianceProbe: + return _ApplianceProbe( + env.get("OPENBLADE_APPLIANCE_URL", ""), + env.get("OPENBLADE_APPLIANCE_USER", ""), + env.get("OPENBLADE_APPLIANCE_PASSWORD", ""), + insecure=env.get("OPENBLADE_APPLIANCE_INSECURE") == "true", + ) + + +def _list_by_id() -> list[str]: + try: + return [os.path.realpath(e.path) + "|" + e.name for e in os.scandir(BY_ID_DIR)] + except OSError: + return [] + + +@dataclass +class Host: + """Everything the checks touch on the host — injected so unit tests use fakes.""" + + env: Mapping[str, str] + runner: SafeRunner + which: Callable[[str], str | None] = shutil.which + exists: Callable[[str], bool] = os.path.exists + readable: Callable[[str], bool] = field(default=lambda p: os.access(p, os.R_OK)) + by_id: Callable[[], list[str]] = _list_by_id + appliance: Callable[[Mapping[str, str]], ApplianceProbe] = _default_appliance + + +def _drives(env: Mapping[str, str]) -> list[str]: + return [d.strip() for d in env.get("OPENBLADE_DRIVE_DEVICES", "").split(",") if d.strip()] + + +def check_armed(host: Host) -> Check: + if host.env.get("OPENBLADE_REAL_HARDWARE_ENABLED"): + return Check( + "armed", + "FAIL", + "OPENBLADE_REAL_HARDWARE_ENABLED is set in the preflight env", + "unset it: preflight is read-only and must never run armed", + ) + return Check("armed", "PASS", "not armed") + + +def check_env(host: Host, lane: Lane) -> list[Check]: + required = [ + "OPENBLADE_CHANGER_DEVICE", + "OPENBLADE_DRIVE_DEVICES", + "OPENBLADE_APPLIANCE_URL", + "OPENBLADE_APPLIANCE_USER", + "OPENBLADE_APPLIANCE_PASSWORD", + ] + out: list[Check] = [] + for var in [*required, "SACRIFICIAL_BARCODES"]: + if host.env.get(var, "").strip(): + # Presence only — values (notably the password) are never printed. + out.append(Check(f"env:{var}", "PASS", "set")) + continue + hard = var in required or lane == "destructive" + where = ( + "runner .env" + if var.startswith("OPENBLADE_") and "APPLIANCE" not in var + else "repo var/secret" + ) + out.append( + Check( + f"env:{var}", + "FAIL" if hard else "WARN", + f"{var} is not set", + f"set {var} ({where})", + ) + ) + return out + + +def check_tools(host: Host) -> list[Check]: + return [ + Check(f"tool:{t}", "PASS", host.which(t) or "") + if host.which(t) + else Check( + f"tool:{t}", "FAIL", f"{t} not on PATH", "install sg3-utils / lsscsi / mtx / LTFS" + ) + for t in REQUIRED_TOOLS + ] + + +def check_devices(host: Host) -> list[Check]: + changer = host.env.get("OPENBLADE_CHANGER_DEVICE", "").strip() + drives = _drives(host.env) + out: list[Check] = [] + for dev in ([changer] if changer else []) + drives: + if not host.exists(dev): + out.append( + Check(f"device:{dev}", "FAIL", "does not exist", "fix the path in the runner .env") + ) + elif not host.readable(dev): + out.append( + Check( + f"device:{dev}", + "FAIL", + "not readable", + "add the runner user to the tape+disk groups", + ) + ) + else: + out.append(Check(f"device:{dev}", "PASS", "exists, readable")) + serials: list[str] = [] + for dev in drives: + res = host.runner.run(["sg_inq", dev]) + inq = parse_sg_inq(res.stdout) if res.success else None + if inq is None or not inq.serial: + out.append( + Check( + f"inquiry:{dev}", + "FAIL", + "sg_inq returned no serial", + "check cabling/permissions", + ) + ) + serials.append("") + continue + out.append( + Check( + f"inquiry:{dev}", + "PASS", + f"{inq.vendor} {inq.product} {inq.revision} sn={inq.serial}", + ) + ) + serials.append(inq.serial) + real = [s for s in serials if s] + dupes = sorted({s for s in real if real.count(s) > 1}) + if dupes: + out.append( + Check( + "serials", + "FAIL", + f"duplicate drive serials {dupes}", + "a drive is listed twice in OPENBLADE_DRIVE_DEVICES", + ) + ) + elif real: + out.append(Check("serials", "PASS", f"{len(real)} unique")) + links = host.by_id() + for serial in real: + if not any(serial in link.rsplit("|", 1)[-1] for link in links): + out.append( + Check( + f"by-id:{serial}", + "FAIL", + f"no {BY_ID_DIR} entry", + "check udev rules for tape devices", + ) + ) + else: + out.append(Check(f"by-id:{serial}", "PASS", "resolves")) + raw_map = host.env.get("OPENBLADE_DRIVE_SERIAL_MAP", "") + if raw_map.strip(): + try: + mapping = dict(parse_drive_serial_map(raw_map)) + except DriveCorrelationError: + out.append( + Check( + "serial-map", + "FAIL", + "OPENBLADE_DRIVE_SERIAL_MAP is malformed", + "use SERIAL:DTE,...", + ) + ) + return out + wrong = [ + f"{s}->{mapping.get(s)} (inquiry DTE {i})" + for i, s in enumerate(serials) + if s and mapping.get(s) != i + ] + if wrong: + out.append( + Check( + "serial-map", + "FAIL", + "disagrees with inquiry order: " + ", ".join(wrong), + "fix DTE order", + ) + ) + else: + out.append(Check("serial-map", "PASS", "agrees with inquiry order")) + return out + + +def check_changer(host: Host) -> tuple[list[Check], MtxStatus | None]: + changer = host.env.get("OPENBLADE_CHANGER_DEVICE", "").strip() + if not changer: + return [Check("changer", "FAIL", "no changer device", "set OPENBLADE_CHANGER_DEVICE")], None + res = host.runner.run(["mtx", "-f", changer, "status"]) + if not res.success: + return [ + Check( + "changer", "FAIL", f"mtx status failed (rc={res.returncode})", "check the sg device" + ) + ], None + status = parse_mtx_status(res.stdout) + want = len(_drives(host.env)) + got = status.drive_count or len(status.drives) + if got != want: + return [ + Check( + "changer", + "FAIL", + f"changer reports {got} drives, env lists {want}", + "fix OPENBLADE_DRIVE_DEVICES", + ) + ], status + return [ + Check("changer", "PASS", f"{got} drives, {status.slot_count or len(status.slots)} slots") + ], status + + +def check_allowlist(host: Host, status: MtxStatus | None, lane: Lane) -> list[Check]: + barcodes = parse_barcodes(host.env.get("SACRIFICIAL_BARCODES")) + bad: Status = "FAIL" if lane == "destructive" else "WARN" + if not barcodes: + return [ + Check( + "allowlist", + bad, + "SACRIFICIAL_BARCODES is empty", + "set the repo var to scratch tapes", + ) + ] + if status is None: + return [ + Check("allowlist", bad, "no inventory to check against", "fix the changer check first") + ] + loaded = {d.barcode.upper() for d in status.drives if d.loaded and d.barcode} + present = loaded | { + s.barcode.upper() for s in [*status.slots, *status.import_export_slots] if s.barcode + } + out: list[Check] = [] + missing = [b for b in barcodes if b not in present] + in_drive = [b for b in barcodes if b in loaded] + if missing: + out.append( + Check("allowlist", bad, f"not in inventory: {missing}", "fix SACRIFICIAL_BARCODES") + ) + if in_drive: + out.append( + Check( + "allowlist", bad, f"loaded in a drive: {in_drive}", "unload them via the UI first" + ) + ) + if not out: + out.append(Check("allowlist", "PASS", f"{len(barcodes)} barcode(s) in slots, none loaded")) + return out + + +def check_appliance(host: Host) -> list[Check]: + env = host.env + out: list[Check] = [] + if env.get("OPENBLADE_APPLIANCE_INSECURE") == "true": + out.append( + Check( + "appliance:tls", + "WARN", + "TLS verification disabled", + "install the appliance CA instead", + ) + ) + if not all( + env.get(v) + for v in ( + "OPENBLADE_APPLIANCE_URL", + "OPENBLADE_APPLIANCE_USER", + "OPENBLADE_APPLIANCE_PASSWORD", + ) + ): + out.append( + Check("appliance", "FAIL", "appliance credentials incomplete", "see env:* checks") + ) + return out + try: + client = host.appliance(env) + code = client.login() + if code != 200: + out.append( + Check("appliance", "FAIL", f"login HTTP {code}", "check user/password and the URL") + ) + return out + client.logout() + except PreflightError as exc: + out.append( + Check("appliance", "FAIL", str(exc), "check OPENBLADE_APPLIANCE_URL and the network") + ) + return out + out.append(Check("appliance", "PASS", "login + logout OK")) + return out + + +def run_preflight(host: Host, lane: Lane) -> list[Check]: + checks = [check_armed(host), *check_env(host, lane), *check_tools(host), *check_devices(host)] + changer_checks, status = check_changer(host) + return [*checks, *changer_checks, *check_allowlist(host, status, lane), *check_appliance(host)] + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m tools.hardware.preflight") + parser.add_argument("--lane", choices=["readonly", "destructive"], required=True) + parser.add_argument("--json", action="store_true", help="emit machine-readable JSON on stdout") + args = parser.parse_args(argv) + lane: Lane = "destructive" if args.lane == "destructive" else "readonly" + checks = run_preflight(Host(env=os.environ, runner=SafeRunner()), lane) + failed = any(c.status == "FAIL" for c in checks) + if args.json: + print( + json.dumps( + {"lane": lane, "ok": not failed, "checks": [asdict(c) for c in checks]}, indent=2 + ) + ) + else: + for c in checks: + print( + f"{c.status:4} {c.name:32} {c.detail}" + + (f" -> {c.fix}" if c.fix and c.status != "PASS" else "") + ) + return EXIT_FAIL if failed else EXIT_OK + + +if __name__ == "__main__": + sys.exit(main())